From 53f010f803e20dd1a237133fe092f3f24dc0598a Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Tue, 13 Jan 2026 13:47:30 -0500 Subject: [PATCH 01/33] Changed name for easier IDE debugging --- batbot/{batbot.py => batbot_cli.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename batbot/{batbot.py => batbot_cli.py} (100%) diff --git a/batbot/batbot.py b/batbot/batbot_cli.py similarity index 100% rename from batbot/batbot.py rename to batbot/batbot_cli.py From 579e88eec9a9dc42c9ce77057dc3b552b6de90e3 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Tue, 13 Jan 2026 13:48:17 -0500 Subject: [PATCH 02/33] Added "fast_mode" pipeline option --- batbot/__init__.py | 9 +- batbot/spectrogram/__init__.py | 367 +++++++++++++++++---------------- 2 files changed, 201 insertions(+), 175 deletions(-) diff --git a/batbot/__init__.py b/batbot/__init__.py index 111778f..25999de 100644 --- a/batbot/__init__.py +++ b/batbot/__init__.py @@ -63,6 +63,7 @@ def pipeline( config=None, # classifier_thresh=classifier.CONFIGS[None]['thresh'], clean=True, + fast_mode=False, ): """ Run the ML pipeline on a given WAV filepath and return the classification results @@ -93,7 +94,7 @@ def pipeline( tuple ( float, list ( dict ) ): classifier score, list of time windows """ # Generate spectrogram - output_paths, metadata_path, metadata = spectrogram.compute(filepath) + output_paths, metadata_path, metadata = spectrogram.compute(filepath, fast_mode=fast_mode) return output_paths, metadata_path @@ -162,6 +163,10 @@ def example(): log.debug(f'Running pipeline on WAV: {wav_filepath}') - results = pipeline(wav_filepath) + import time + start_time = time.time() + results = pipeline(wav_filepath, fast_mode=False) + stop_time = time.time() + print('Example pipeline completed in {} seconds.'.format(stop_time - start_time)) log.debug(results) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index baf5537..ea4dfc2 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -133,10 +133,10 @@ def plot_histogram( # if ignore_zeros: # assert hist[0] == 0 - hist_original = hist.copy() + if output_path: hist_original = hist.copy() if smoothing: hist = gaussian_filter1d(hist, smoothing, mode='nearest') - hist_original = (hist_original / hist_original.max()) * hist.max() + if output_path: hist_original = (hist_original / hist_original.max()) * hist.max() mode_ = np.argmax(hist) # histogram mode @@ -236,9 +236,9 @@ def generate_waveplot( return waveplot - +@lp def load_stft( - wav_filepath, sr=250e3, n_fft=512, window='blackmanharris', win_length=256, hop_length=16 + wav_filepath, sr=250e3, n_fft=512, window='blackmanharris', win_length=256, hop_length=16, fast_mode=False ): assert exists(wav_filepath) log.debug(f'Computing spectrogram on {wav_filepath}') @@ -282,12 +282,15 @@ def load_stft( stft_db = stft_db[min_index : max_index + 1, :] bands = bands[min_index : max_index + 1] - waveplot = generate_waveplot(waveform, stft_db, hop_length=hop_length) + if fast_mode: + waveplot = [] + else: + waveplot = generate_waveplot(waveform, stft_db, hop_length=hop_length) return stft_db, waveplot, sr, bands, duration, min_index, time_vec - -def gain_stft(stft_db, gain_db=80.0, autogain_stddev=5.0): +@lp +def gain_stft(stft_db, gain_db=80.0, autogain_stddev=5.0, fast_mode=False): # Subtract per-frequency median DB med = np.median(stft_db, axis=1).reshape(-1, 1) stft_db -= med @@ -298,14 +301,15 @@ def gain_stft(stft_db, gain_db=80.0, autogain_stddev=5.0): assert stft_db.max() == 0 stft_db += gain_db - # Calculate the non-zero median DB and MAD - # autogain signal if (median - alpha * deviation) is higher than provided gain - temp = stft_db[stft_db > 0] - med_db = np.median(temp) - std_db = scipy.stats.median_abs_deviation(temp, axis=None, scale='normal') - autogain_value = med_db - (autogain_stddev * std_db) - if autogain_value > 0: - stft_db -= autogain_value + if not fast_mode: + # Calculate the non-zero median DB and MAD + # autogain signal if (median - alpha * deviation) is higher than provided gain + temp = stft_db[stft_db > 0] + med_db = np.median(temp) + std_db = scipy.stats.median_abs_deviation(temp, axis=None, scale='normal') + autogain_value = med_db - (autogain_stddev * std_db) + if autogain_value > 0: + stft_db -= autogain_value # Clip values below zero stft_db = np.clip(stft_db, 0.0, None) @@ -388,12 +392,12 @@ def create_coarse_candidates(stft_db, window, stride, threshold_stddev=3.0): return candidates, candidate_dbs - +@lp def filter_candidates_to_ranges( - stft_db, candidates, window=16, skew_stddev=2.0, area_percent=0.10, output_path=None + stft_db, candidates, window=16, skew_stddev=2.0, area_percent=0.10, output_path=None, fast_mode=False ): # Filter the candidates based on their distribution skewness - stride_ = 2 + stride_ = 2 if not fast_mode else 16 buffer = int(round(window / stride_ / 2)) reject_idxs = [] @@ -1300,7 +1304,7 @@ def calculate_harmonic_and_echo_flags( @lp def compute_wrapper( - wav_filepath, annotations=None, output_folder='.', bitdepth=16, debug=False, **kwargs + wav_filepath, annotations=None, output_folder='.', bitdepth=16, fast_mode=False, debug=False, **kwargs ): """ Compute the spectrograms for a given input WAV and saves them to disk. @@ -1324,6 +1328,7 @@ def compute_wrapper( """ base = splitext(basename(wav_filepath))[0] + if fast_mode: bitdepth = 8 assert bitdepth in [8, 16] dtype = np.uint8 if bitdepth == 8 else np.uint16 @@ -1332,10 +1337,10 @@ def compute_wrapper( debug_path = get_debug_path(output_folder, wav_filepath, enabled=debug) # Load the spectrogram from a WAV file on disk - stft_db, waveplot, sr, bands, duration, freq_offset, time_vec = load_stft(wav_filepath) + stft_db, waveplot, sr, bands, duration, freq_offset, time_vec = load_stft(wav_filepath, fast_mode=fast_mode) # Apply a dynamic range to a fixed dB range - stft_db = gain_stft(stft_db) + stft_db = gain_stft(stft_db, fast_mode=fast_mode) # Bin the floating point data to X-bit integers (X=8 or X=16) stft_db = normalize_stft(stft_db, None, dtype) @@ -1351,20 +1356,25 @@ def compute_wrapper( # # Save the spectrogram image to disk # cv2.imwrite('debug.tif', stft_db, [cv2.IMWRITE_TIFF_COMPRESSION, 1]) - # Plot the histogram, ignoring any non-zero values (will no-op if output_path is None) - global_med_db, global_std_db, global_peak_db = plot_histogram( - stft_db, ignore_zeros=True, smoothing=512, output_path=debug_path - ) - # Estimate a global threshold for finding the edges of bat call contours - global_threshold_std = 2.0 - global_threshold = global_peak_db - global_threshold_std * global_std_db + if not fast_mode: + # Plot the histogram, ignoring any non-zero values (will no-op if output_path is None) + global_med_db, global_std_db, global_peak_db = plot_histogram( + stft_db, ignore_zeros=True, smoothing=512, output_path=debug_path + ) + # Estimate a global threshold for finding the edges of bat call contours + global_threshold_std = 2.0 + global_threshold = global_peak_db - global_threshold_std * global_std_db + else: + # Fast mode skips bat call segmentation + global_threshold = 0.0 # Get a distribution of the max candidate locations - window, stride = calculate_window_and_stride(stft_db, duration, time_vec=time_vec) + strides_per_window = 3 if not fast_mode else 6 + window, stride = calculate_window_and_stride(stft_db, duration, strides_per_window=strides_per_window, time_vec=time_vec) candidates, candidate_max_dbs = create_coarse_candidates(stft_db, window, stride) # Filter all candidates to the ranges that have a substantial right-side skew - ranges, reject_idxs = filter_candidates_to_ranges(stft_db, candidates, output_path=debug_path) + ranges, reject_idxs = filter_candidates_to_ranges(stft_db, candidates, output_path=debug_path, fast_mode=fast_mode) # Add in user-specified annotations to ranges if annotations: @@ -1379,172 +1389,183 @@ def compute_wrapper( # Plot the chirp candidates (will no-op if output_path is None) plot_chirp_candidates(stft_db, candidate_max_dbs, ranges, reject_idxs, output_path=debug_path) - # Tighten the ranges by looking for substantial right-side skew (use stride for a smaller sampling window) - ranges = tighten_ranges(stft_db, ranges, stride, duration, output_path=debug_path) + if fast_mode: + # Apply reduced processing without segment refinement or metadata calculation + segments = {'stft_db': [] } + # Remove a fraction of the window length when not doing call segmentation + crop_length = max(0, int(round(0.75 * window - 1))) + for start, stop in ranges: + segments['stft_db'].append(stft_db[:, start + crop_length: stop - crop_length]) + metas = {} - # Extract chirp metrics and metadata - segments = { - 'stft_db': [], - 'waveplot': [], - 'costs': [], - 'canvas': [], - } - metas = [] - for index, (start, stop) in tqdm.tqdm(list(enumerate(ranges))): - segment = stft_db[:, start:stop] + else: - # Step 0.1 - Debugging setup and find peak amplitude (will return None if disabled) - canvas = create_contour_debug_canvas(segment, index, output_path=debug_path) + # Tighten the ranges by looking for substantial right-side skew (use stride for a smaller sampling window) + ranges = tighten_ranges(stft_db, ranges, stride, duration, output_path=debug_path) - # Step 0.2 - Find the location(s) of peak amplitude - max_locations = find_max_locations(segment) + # Extract chirp metrics and metadata + segments = { + 'stft_db': [], + 'waveplot': [], + 'costs': [], + 'canvas': [], + } + metas = [] + for index, (start, stop) in tqdm.tqdm(list(enumerate(ranges))): + segment = stft_db[:, start:stop] - # Step 1 - Scale with PDF - segment, peak_db, peak_db_std = scale_pdf_contour(segment, index, output_path=debug_path) - if None in {peak_db, peak_db_std}: - continue + # Step 0.1 - Debugging setup and find peak amplitude (will return None if disabled) + canvas = create_contour_debug_canvas(segment, index, output_path=debug_path) - # Step 2 - Apply median filtering to contour - segment = filter_contour(segment, index, output_path=debug_path) + # Step 0.2 - Find the location(s) of peak amplitude + max_locations = find_max_locations(segment) - # Step 3 - Apply Morphology Open to contour - segment = morph_open_contour(segment, index, output_path=debug_path) + # Step 1 - Scale with PDF + segment, peak_db, peak_db_std = scale_pdf_contour(segment, index, output_path=debug_path) + if None in {peak_db, peak_db_std}: + continue - # Step 4 - Normalize contour - segment = normalize_contour(segment, index, output_path=debug_path) + # Step 2 - Apply median filtering to contour + segment = filter_contour(segment, index, output_path=debug_path) - # # Step 5 (OLD) - Threshold contour - # segment, med_db, std_db, peak_db = threshold_contour(segment, index, output_path=debug_path) + # Step 3 - Apply Morphology Open to contour + segment = morph_open_contour(segment, index, output_path=debug_path) - # Step 5 - Find primary contour that contains max amplitude - # (To use a local instead of global threshold, remove the threshold argument here) - segmentmask, peak, segment_threshold = find_contour_and_peak( - segment, - index, - max_locations, - peak_db, - peak_db_std, - output_path=debug_path, - threshold=global_threshold, - ) + # Step 4 - Normalize contour + segment = normalize_contour(segment, index, output_path=debug_path) - if peak is None: - continue + # # Step 5 (OLD) - Threshold contour + # segment, med_db, std_db, peak_db = threshold_contour(segment, index, output_path=debug_path) - # Step 6 - Create final segmentmask - segmentmask = refine_segmentmask(segmentmask, index, output_path=debug_path) + # Step 5 - Find primary contour that contains max amplitude + # (To use a local instead of global threshold, remove the threshold argument here) + segmentmask, peak, segment_threshold = find_contour_and_peak( + segment, + index, + max_locations, + peak_db, + peak_db_std, + output_path=debug_path, + threshold=global_threshold, + ) - # # Step 6 (OLD) - Find the contour with the (most) max amplitude location(s) - # valid, segmentmask, peak = find_contour_connected_components(segment, index, max_locations, output_path=debug_path) - # # Step 6 (OLD) - Refine contour by removing any harmonic or echo - # segmentmask, peak = refine_contour(segment_, index, max_locations, segmentmask, peak, output_path=debug_path) + if peak is None: + continue - # Step 7 - Calculate the first order harmonic and echo region - harmonic = find_harmonic(segmentmask, index, freq_offset, output_path=debug_path) - echo = find_echo(segmentmask, index, output_path=debug_path) + # Step 6 - Create final segmentmask + segmentmask = refine_segmentmask(segmentmask, index, output_path=debug_path) - original = stft_db[:, start:stop] - harmonic_flag, hamonic_peak, echo_flag, echo_peak = calculate_harmonic_and_echo_flags( - original, index, segmentmask, harmonic, echo, canvas, output_path=debug_path - ) + # # Step 6 (OLD) - Find the contour with the (most) max amplitude location(s) + # valid, segmentmask, peak = find_contour_connected_components(segment, index, max_locations, output_path=debug_path) + # # Step 6 (OLD) - Refine contour by removing any harmonic or echo + # segmentmask, peak = refine_contour(segment_, index, max_locations, segmentmask, peak, output_path=debug_path) - # Remove harmonic and echo from segmentation - segment = remove_harmonic_and_echo( - segment, index, harmonic, echo, global_threshold, output_path=debug_path - ) + # Step 7 - Calculate the first order harmonic and echo region + harmonic = find_harmonic(segmentmask, index, freq_offset, output_path=debug_path) + echo = find_echo(segmentmask, index, output_path=debug_path) - # Step 8 - Calculate the A* cost grid and bat call start/end points - costs, grid, call_begin, call_end, boundary = calculate_astar_grid_and_endpoints( - segment, index, segmentmask, peak, canvas, output_path=debug_path - ) - top, bottom, left, right = boundary + original = stft_db[:, start:stop] + harmonic_flag, hamonic_peak, echo_flag, echo_peak = calculate_harmonic_and_echo_flags( + original, index, segmentmask, harmonic, echo, canvas, output_path=debug_path + ) - # Skip chirp if the extracted path covers a small duration or bandwidth - bandwidth, duration_, significant = significant_contour_path( - call_begin, call_end, y_step_freq, x_step_ms - ) - if not significant: - continue + # Remove harmonic and echo from segmentation + segment = remove_harmonic_and_echo( + segment, index, harmonic, echo, global_threshold, output_path=debug_path + ) - # Step 9 - Extract optimal path from start to end using the cost grid - path = extract_contour_path( - grid, call_begin, call_end, canvas, index, output_path=debug_path - ) + # Step 8 - Calculate the A* cost grid and bat call start/end points + costs, grid, call_begin, call_end, boundary = calculate_astar_grid_and_endpoints( + segment, index, segmentmask, peak, canvas, output_path=debug_path + ) + top, bottom, left, right = boundary - # Step 10 - Extract contour keypoints - path_smoothed, (knee, fc, heel), slopes = extract_contour_keypoints( - path, canvas, index, peak, output_path=debug_path - ) + # Skip chirp if the extracted path covers a small duration or bandwidth + bandwidth, duration_, significant = significant_contour_path( + call_begin, call_end, y_step_freq, x_step_ms + ) + if not significant: + continue - # Step 11 - Collect chirp metadata - metadata = { - 'curve.(hz,ms)': [ - ( - bands[y], - (start + x) * x_step_ms, - ) - for y, x in path_smoothed - ], - 'start.ms': (start + left) * x_step_ms, - 'end.ms': (start + right) * x_step_ms, - 'duration.ms': (right - left) * x_step_ms, - 'threshold.amp': int(round(255.0 * (segment_threshold / np.iinfo(stft_db.dtype).max))), - 'peak f.ms': (start + peak[1]) * x_step_ms, - 'fc.ms': (start + bands[fc[1]]) * x_step_ms, - 'hi fc:knee.ms': (start + bands[knee[1]]) * x_step_ms, - 'lo fc:heel.ms': (start + bands[heel[1]]) * x_step_ms, - 'bandwidth.hz': bandwidth, - 'hi f.hz': bands[top], - 'lo f.hz': bands[bottom], - 'peak f.hz': bands[peak[0]], - 'fc.hz': bands[fc[0]], - 'hi fc:knee.hz': bands[knee[0]], - 'lo fc:heel.hz': bands[heel[0]], - 'harmonic.flag': harmonic_flag, - 'harmonic peak f.ms': (start + hamonic_peak[1]) * x_step_ms if harmonic_flag else None, - 'harmonic peak f.hz': bands[hamonic_peak[0]] if harmonic_flag else None, - 'echo.flag': echo_flag, - 'echo peak f.ms': (start + echo_peak[1]) * x_step_ms if echo_flag else None, - 'echo peak f.hz': bands[echo_peak[0]] if echo_flag else None, - } - metadata.update(slopes) + # Step 9 - Extract optimal path from start to end using the cost grid + path = extract_contour_path( + grid, call_begin, call_end, canvas, index, output_path=debug_path + ) - # Normalize values - for key, value in list(metadata.items()): - if value is None: - continue - if key.endswith('.ms'): - metadata[key] = round(float(value), 3) - if key.endswith('.hz'): - metadata[key] = int(round(value)) - if key.endswith('.flag'): - metadata[key] = bool(value) - if key.endswith('.y_px/x_px'): - key_ = key.replace('.y_px/x_px', '.khz/ms') - metadata[key_] = round(float(value * ((y_step_freq / 1000.0) / x_step_ms)), 3) - metadata.pop(key) - if key.endswith('.(hz,ms)'): - metadata[key] = [ + # Step 10 - Extract contour keypoints + path_smoothed, (knee, fc, heel), slopes = extract_contour_keypoints( + path, canvas, index, peak, output_path=debug_path + ) + + # Step 11 - Collect chirp metadata + metadata = { + 'curve.(hz,ms)': [ ( - int(round(val1)), - round(float(val2), 3), + bands[y], + (start + x) * x_step_ms, ) - for val1, val2 in value - ] - - metas.append(metadata) - - # Trim segment around the bat call with a small buffer - buffer_ms = 1.0 - buffer_pix = int(round(buffer_ms / x_step_ms)) - trim_begin = max(0, min(segment.shape[1], call_begin[1] - buffer_pix)) - trim_end = max(0, min(segment.shape[1], call_end[1] + buffer_pix)) - - segments['stft_db'].append(stft_db[:, start + trim_begin : start + trim_end]) - segments['waveplot'].append(waveplot[:, start + trim_begin : start + trim_end]) - segments['costs'].append(costs[:, trim_begin:trim_end]) - if debug_path: - segments['canvas'].append(canvas[:, trim_begin:trim_end]) + for y, x in path_smoothed + ], + 'start.ms': (start + left) * x_step_ms, + 'end.ms': (start + right) * x_step_ms, + 'duration.ms': (right - left) * x_step_ms, + 'threshold.amp': int(round(255.0 * (segment_threshold / np.iinfo(stft_db.dtype).max))), + 'peak f.ms': (start + peak[1]) * x_step_ms, + 'fc.ms': (start + bands[fc[1]]) * x_step_ms, + 'hi fc:knee.ms': (start + bands[knee[1]]) * x_step_ms, + 'lo fc:heel.ms': (start + bands[heel[1]]) * x_step_ms, + 'bandwidth.hz': bandwidth, + 'hi f.hz': bands[top], + 'lo f.hz': bands[bottom], + 'peak f.hz': bands[peak[0]], + 'fc.hz': bands[fc[0]], + 'hi fc:knee.hz': bands[knee[0]], + 'lo fc:heel.hz': bands[heel[0]], + 'harmonic.flag': harmonic_flag, + 'harmonic peak f.ms': (start + hamonic_peak[1]) * x_step_ms if harmonic_flag else None, + 'harmonic peak f.hz': bands[hamonic_peak[0]] if harmonic_flag else None, + 'echo.flag': echo_flag, + 'echo peak f.ms': (start + echo_peak[1]) * x_step_ms if echo_flag else None, + 'echo peak f.hz': bands[echo_peak[0]] if echo_flag else None, + } + metadata.update(slopes) + + # Normalize values + for key, value in list(metadata.items()): + if value is None: + continue + if key.endswith('.ms'): + metadata[key] = round(float(value), 3) + if key.endswith('.hz'): + metadata[key] = int(round(value)) + if key.endswith('.flag'): + metadata[key] = bool(value) + if key.endswith('.y_px/x_px'): + key_ = key.replace('.y_px/x_px', '.khz/ms') + metadata[key_] = round(float(value * ((y_step_freq / 1000.0) / x_step_ms)), 3) + metadata.pop(key) + if key.endswith('.(hz,ms)'): + metadata[key] = [ + ( + int(round(val1)), + round(float(val2), 3), + ) + for val1, val2 in value + ] + + metas.append(metadata) + + # Trim segment around the bat call with a small buffer + buffer_ms = 1.0 + buffer_pix = int(round(buffer_ms / x_step_ms)) + trim_begin = max(0, min(segment.shape[1], call_begin[1] - buffer_pix)) + trim_end = max(0, min(segment.shape[1], call_end[1] + buffer_pix)) + + segments['stft_db'].append(stft_db[:, start + trim_begin : start + trim_end]) + segments['waveplot'].append(waveplot[:, start + trim_begin : start + trim_end]) + segments['costs'].append(costs[:, trim_begin:trim_end]) + if debug_path: + segments['canvas'].append(canvas[:, trim_begin:trim_end]) # Concatenate extracted, trimmed segments and other matrices for key in list(segments.keys()): From 28943ac335e80f5817bc6ee83bb27d59be3b1785 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Tue, 13 Jan 2026 13:56:14 -0500 Subject: [PATCH 03/33] Linting --- batbot/__init__.py | 1 + batbot/spectrogram/__init__.py | 64 ++++++++++++++++++++++++++-------- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/batbot/__init__.py b/batbot/__init__.py index 25999de..2d1311b 100644 --- a/batbot/__init__.py +++ b/batbot/__init__.py @@ -164,6 +164,7 @@ def example(): log.debug(f'Running pipeline on WAV: {wav_filepath}') import time + start_time = time.time() results = pipeline(wav_filepath, fast_mode=False) stop_time = time.time() diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index ea4dfc2..a7fe364 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -133,10 +133,12 @@ def plot_histogram( # if ignore_zeros: # assert hist[0] == 0 - if output_path: hist_original = hist.copy() + if output_path: + hist_original = hist.copy() if smoothing: hist = gaussian_filter1d(hist, smoothing, mode='nearest') - if output_path: hist_original = (hist_original / hist_original.max()) * hist.max() + if output_path: + hist_original = (hist_original / hist_original.max()) * hist.max() mode_ = np.argmax(hist) # histogram mode @@ -236,9 +238,16 @@ def generate_waveplot( return waveplot + @lp def load_stft( - wav_filepath, sr=250e3, n_fft=512, window='blackmanharris', win_length=256, hop_length=16, fast_mode=False + wav_filepath, + sr=250e3, + n_fft=512, + window='blackmanharris', + win_length=256, + hop_length=16, + fast_mode=False, ): assert exists(wav_filepath) log.debug(f'Computing spectrogram on {wav_filepath}') @@ -289,6 +298,7 @@ def load_stft( return stft_db, waveplot, sr, bands, duration, min_index, time_vec + @lp def gain_stft(stft_db, gain_db=80.0, autogain_stddev=5.0, fast_mode=False): # Subtract per-frequency median DB @@ -392,9 +402,16 @@ def create_coarse_candidates(stft_db, window, stride, threshold_stddev=3.0): return candidates, candidate_dbs + @lp def filter_candidates_to_ranges( - stft_db, candidates, window=16, skew_stddev=2.0, area_percent=0.10, output_path=None, fast_mode=False + stft_db, + candidates, + window=16, + skew_stddev=2.0, + area_percent=0.10, + output_path=None, + fast_mode=False, ): # Filter the candidates based on their distribution skewness stride_ = 2 if not fast_mode else 16 @@ -1304,7 +1321,13 @@ def calculate_harmonic_and_echo_flags( @lp def compute_wrapper( - wav_filepath, annotations=None, output_folder='.', bitdepth=16, fast_mode=False, debug=False, **kwargs + wav_filepath, + annotations=None, + output_folder='.', + bitdepth=16, + fast_mode=False, + debug=False, + **kwargs, ): """ Compute the spectrograms for a given input WAV and saves them to disk. @@ -1328,7 +1351,8 @@ def compute_wrapper( """ base = splitext(basename(wav_filepath))[0] - if fast_mode: bitdepth = 8 + if fast_mode: + bitdepth = 8 assert bitdepth in [8, 16] dtype = np.uint8 if bitdepth == 8 else np.uint16 @@ -1337,7 +1361,9 @@ def compute_wrapper( debug_path = get_debug_path(output_folder, wav_filepath, enabled=debug) # Load the spectrogram from a WAV file on disk - stft_db, waveplot, sr, bands, duration, freq_offset, time_vec = load_stft(wav_filepath, fast_mode=fast_mode) + stft_db, waveplot, sr, bands, duration, freq_offset, time_vec = load_stft( + wav_filepath, fast_mode=fast_mode + ) # Apply a dynamic range to a fixed dB range stft_db = gain_stft(stft_db, fast_mode=fast_mode) @@ -1370,11 +1396,15 @@ def compute_wrapper( # Get a distribution of the max candidate locations strides_per_window = 3 if not fast_mode else 6 - window, stride = calculate_window_and_stride(stft_db, duration, strides_per_window=strides_per_window, time_vec=time_vec) + window, stride = calculate_window_and_stride( + stft_db, duration, strides_per_window=strides_per_window, time_vec=time_vec + ) candidates, candidate_max_dbs = create_coarse_candidates(stft_db, window, stride) # Filter all candidates to the ranges that have a substantial right-side skew - ranges, reject_idxs = filter_candidates_to_ranges(stft_db, candidates, output_path=debug_path, fast_mode=fast_mode) + ranges, reject_idxs = filter_candidates_to_ranges( + stft_db, candidates, output_path=debug_path, fast_mode=fast_mode + ) # Add in user-specified annotations to ranges if annotations: @@ -1391,11 +1421,11 @@ def compute_wrapper( if fast_mode: # Apply reduced processing without segment refinement or metadata calculation - segments = {'stft_db': [] } + segments = {'stft_db': []} # Remove a fraction of the window length when not doing call segmentation crop_length = max(0, int(round(0.75 * window - 1))) for start, stop in ranges: - segments['stft_db'].append(stft_db[:, start + crop_length: stop - crop_length]) + segments['stft_db'].append(stft_db[:, start + crop_length : stop - crop_length]) metas = {} else: @@ -1421,7 +1451,9 @@ def compute_wrapper( max_locations = find_max_locations(segment) # Step 1 - Scale with PDF - segment, peak_db, peak_db_std = scale_pdf_contour(segment, index, output_path=debug_path) + segment, peak_db, peak_db_std = scale_pdf_contour( + segment, index, output_path=debug_path + ) if None in {peak_db, peak_db_std}: continue @@ -1509,7 +1541,9 @@ def compute_wrapper( 'start.ms': (start + left) * x_step_ms, 'end.ms': (start + right) * x_step_ms, 'duration.ms': (right - left) * x_step_ms, - 'threshold.amp': int(round(255.0 * (segment_threshold / np.iinfo(stft_db.dtype).max))), + 'threshold.amp': int( + round(255.0 * (segment_threshold / np.iinfo(stft_db.dtype).max)) + ), 'peak f.ms': (start + peak[1]) * x_step_ms, 'fc.ms': (start + bands[fc[1]]) * x_step_ms, 'hi fc:knee.ms': (start + bands[knee[1]]) * x_step_ms, @@ -1522,7 +1556,9 @@ def compute_wrapper( 'hi fc:knee.hz': bands[knee[0]], 'lo fc:heel.hz': bands[heel[0]], 'harmonic.flag': harmonic_flag, - 'harmonic peak f.ms': (start + hamonic_peak[1]) * x_step_ms if harmonic_flag else None, + 'harmonic peak f.ms': ( + (start + hamonic_peak[1]) * x_step_ms if harmonic_flag else None + ), 'harmonic peak f.hz': bands[hamonic_peak[0]] if harmonic_flag else None, 'echo.flag': echo_flag, 'echo peak f.ms': (start + echo_peak[1]) * x_step_ms if echo_flag else None, From 1c794dc10da8235da682997a448c9466563bcad6 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Tue, 13 Jan 2026 13:57:29 -0500 Subject: [PATCH 04/33] Turned off line_profiler for speed --- batbot/spectrogram/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index a7fe364..7c7a3c1 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -239,7 +239,7 @@ def generate_waveplot( return waveplot -@lp +# @lp def load_stft( wav_filepath, sr=250e3, @@ -299,7 +299,7 @@ def load_stft( return stft_db, waveplot, sr, bands, duration, min_index, time_vec -@lp +# @lp def gain_stft(stft_db, gain_db=80.0, autogain_stddev=5.0, fast_mode=False): # Subtract per-frequency median DB med = np.median(stft_db, axis=1).reshape(-1, 1) @@ -403,7 +403,7 @@ def create_coarse_candidates(stft_db, window, stride, threshold_stddev=3.0): return candidates, candidate_dbs -@lp +# @lp def filter_candidates_to_ranges( stft_db, candidates, @@ -1319,7 +1319,7 @@ def calculate_harmonic_and_echo_flags( return harmonic_flag, harmonic_peak, echo_flag, echo_peak -@lp +# @lp def compute_wrapper( wav_filepath, annotations=None, From 4ffe5a865d0629df98e8096879b35619440be455 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Tue, 13 Jan 2026 19:44:57 -0500 Subject: [PATCH 05/33] Fast mode improvements - combine coarse candidates before filtering --- batbot/spectrogram/__init__.py | 38 ++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index 7c7a3c1..cd7b9dd 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -43,6 +43,7 @@ def compute(*args, **kwargs): def get_islands(data): + # Find all islands of contiguous same elements with island length 2 or more mask = np.r_[np.diff(data) == 0, False] mask_ = np.concatenate(([False], mask, [False])) idx_ = np.flatnonzero(mask_[1:] != mask_[:-1]) @@ -414,7 +415,7 @@ def filter_candidates_to_ranges( fast_mode=False, ): # Filter the candidates based on their distribution skewness - stride_ = 2 if not fast_mode else 16 + stride_ = 2 if not fast_mode else 4 buffer = int(round(window / stride_ / 2)) reject_idxs = [] @@ -440,6 +441,9 @@ def filter_candidates_to_ranges( islands = get_islands(skews) area = float(max([val.sum() for val in islands])) area /= len(skews) + if area == 0.0 and sum(skews) >= 1: + # handle edge case with single-element islands + area = 1.0 / len(skews) if area >= area_percent: ranges.append((start, stop)) @@ -1395,11 +1399,31 @@ def compute_wrapper( global_threshold = 0.0 # Get a distribution of the max candidate locations - strides_per_window = 3 if not fast_mode else 6 + # Normal mode uses a relatively large window and little overlap + # Fast mode uses a relatively small window and lots of overlap, since it skips range tightening step + strides_per_window = 3 if not fast_mode else 16 + window_size_ms = 12 if not fast_mode else 3 + threshold_stddev = 3.0 if not fast_mode else 4.5 window, stride = calculate_window_and_stride( - stft_db, duration, strides_per_window=strides_per_window, time_vec=time_vec + stft_db, + duration, + window_size_ms=window_size_ms, + strides_per_window=strides_per_window, + time_vec=time_vec, ) - candidates, candidate_max_dbs = create_coarse_candidates(stft_db, window, stride) + candidates, candidate_max_dbs = create_coarse_candidates( + stft_db, window, stride, threshold_stddev=threshold_stddev + ) + + if fast_mode: + # combine candidates for efficiency and remove very short candidates (likely noise) + tmp_ranges = [(x, y) for _, x, y in candidates] + tmp_ranges = merge_ranges(tmp_ranges, stft_db.shape[1]) + candidate_lengths = np.array([y - x for x, y in tmp_ranges]) + length_thresh = window * 1.5 + idx_remove = candidate_lengths < length_thresh + candidates = [(ii, x, y) for ii, (x, y) in enumerate(tmp_ranges) if not idx_remove[ii]] + candidate_max_dbs = [] # Filter all candidates to the ranges that have a substantial right-side skew ranges, reject_idxs = filter_candidates_to_ranges( @@ -1421,11 +1445,13 @@ def compute_wrapper( if fast_mode: # Apply reduced processing without segment refinement or metadata calculation + segments = {'stft_db': []} # Remove a fraction of the window length when not doing call segmentation - crop_length = max(0, int(round(0.75 * window - 1))) + crop_length_l = max(0, int(round(0.15 * window - 1))) + crop_length_r = max(0, int(round(0.45 * window - 1))) for start, stop in ranges: - segments['stft_db'].append(stft_db[:, start + crop_length : stop - crop_length]) + segments['stft_db'].append(stft_db[:, start + crop_length_l : stop - crop_length_r]) metas = {} else: From e13439bcb70e5a3c7b99c05317631496146768cf Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Fri, 16 Jan 2026 14:20:27 -0500 Subject: [PATCH 06/33] Don't print line_profiler in fast mode --- batbot/spectrogram/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index cd7b9dd..7fd4c1d 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -38,7 +38,8 @@ def compute(*args, **kwargs): retval = compute_wrapper(*args, **kwargs) - lp.print_stats() + if not kwargs.get('fast_mode', True): + lp.print_stats() return retval @@ -420,7 +421,7 @@ def filter_candidates_to_ranges( reject_idxs = [] ranges = [] - for index, (idx, start, stop) in tqdm.tqdm(list(enumerate(candidates))): + for index, (idx, start, stop) in tqdm.tqdm(list(enumerate(candidates)), disable=fast_mode): # Extract the candidate window of the STFT candidate = stft_db[:, start:stop] @@ -1695,9 +1696,12 @@ def compute_wrapper( output_paths = [] compressed_paths = [] - datas = [ - (output_paths, 'jpg', stft_db), - ] + if not fast_mode: + datas = [ + (output_paths, 'jpg', stft_db), + ] + else: + datas = [] if 'stft_db' in segments: datas += [ (compressed_paths, 'compressed.jpg', segments['stft_db']), From f3a639d0c33ab657c76255f831a83a930e9f4bad Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Fri, 16 Jan 2026 14:20:58 -0500 Subject: [PATCH 07/33] Added batch preprocessing to make compressed spectrograms --- batbot/batbot_cli.py | 73 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/batbot/batbot_cli.py b/batbot/batbot_cli.py index b47eb61..d0308e6 100755 --- a/batbot/batbot_cli.py +++ b/batbot/batbot_cli.py @@ -2,14 +2,18 @@ """ CLI for BatBot """ +from glob import glob import json from os.path import exists +import warnings import click import batbot from batbot import log +from tqdm import tqdm + def pipeline_filepath_validator(ctx, param, value): if not exists(value): @@ -98,6 +102,74 @@ def pipeline( else: print(data) +@click.command('preprocess') +@click.argument( + 'filepaths', + nargs=-1, + type=str, +) +# @click.option( +# '--output-dir', +# help='Processed file output directory. Defaults to current working directory.', +# default='.', +# type=str, +# ) +@click.option( + '--metadata', '-m', + help='Use a much slower version of the pipeline which increases spectogram compression quality and outputs additional bat call metadata.', + is_flag=True, +) +@click.option( + '--output-json', + help='Path to output JSON (if unspecified, output file locations are printed to screen)', + default=None, + type=str, +) +def preprocess(filepaths, metadata, output_json): + """Generate compressed spectrogram images for wav files into the current working directory. + Takes one or more space separated arguments of filepaths to process. + Filepaths can use wildcards ** for folders and/or * within filenames (if ** wildcard is used, + will recursively search through all subfolders). + """ + all_filepaths = [] + for file in filepaths: + all_filepaths.extend(glob(file, recursive=True)) + # remove any repeats + all_filepaths = sorted(list(set(all_filepaths))) + + if len(all_filepaths) == 0: + print('Found no files given filepaths input {}'.format(filepaths)) + return + + print('Running preprocessing on {} located files'.format(len(all_filepaths))) + print('\tFast processing mode {}'.format('OFF' if metadata else 'ON')) + print('\tName of first file to process: {}'.format(all_filepaths[0])) + if len(all_filepaths) > 2: + print('\tName of last file to process: {}'.format(all_filepaths[-1])) + + data = {'output_path':[], 'metadata_paths':[]} + for file in tqdm(all_filepaths, desc='Preprocessing files', total=len(all_filepaths)): + try: + output_paths, metadata_path = batbot.pipeline(file, fast_mode=(not metadata)) #, extra_arg=True) + data['output_path'].extend(output_paths) + if metadata: + data['metadata_path'].extend(metadata_path) + except: + warnings.warn('WARNING: Pipeline failed for file {}'.format(file)) + + if output_json is None: + print('Processed output paths:') + print(data['output_path']) + if metadata: + print('Processed metadata paths:') + print(data['metadata_path']) + else: + with open(output_json, 'w') as outfile: + json.dump(data, outfile) + print('Complete.') + + return data + @click.command('batch') @click.argument( @@ -189,6 +261,7 @@ def cli(): cli.add_command(fetch) cli.add_command(pipeline) +cli.add_command(preprocess) cli.add_command(batch) cli.add_command(example) From bd210a6d9186b565dd5eb0655b5c4d865ce3de29 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Mon, 26 Jan 2026 19:05:58 -0500 Subject: [PATCH 08/33] Added parallel pipeline which handles multiple input files. Added out filename stem input to pipelines --- batbot/__init__.py | 130 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 122 insertions(+), 8 deletions(-) diff --git a/batbot/__init__.py b/batbot/__init__.py index 2d1311b..56cebfc 100644 --- a/batbot/__init__.py +++ b/batbot/__init__.py @@ -17,10 +17,13 @@ output_paths, metadata_path, metadata = spectrogram.compute(filepath) """ -from os.path import exists, join +import concurrent.futures +from os.path import exists, join, split, splitext +from multiprocessing import Manager from pathlib import Path import pooch +from tqdm import tqdm from batbot import utils @@ -60,10 +63,10 @@ def fetch(pull=False, config=None): def pipeline( filepath, - config=None, - # classifier_thresh=classifier.CONFIGS[None]['thresh'], - clean=True, + out_file_stem=None, fast_mode=False, + force_overwrite=False, + quiet=False, ): """ Run the ML pipeline on a given WAV filepath and return the classification results @@ -93,11 +96,121 @@ def pipeline( Returns: tuple ( float, list ( dict ) ): classifier score, list of time windows """ + if out_file_stem is None: + # default to writing directly to working directory + out_file_stem = splitext(split(filepath)[-1])[0] # Generate spectrogram - output_paths, metadata_path, metadata = spectrogram.compute(filepath, fast_mode=fast_mode) + output_paths, compressed_paths, metadata_path, metadata = spectrogram.compute(filepath, + out_file_stem=out_file_stem, + fast_mode=fast_mode, + force_overwrite=force_overwrite, + quiet=quiet, + ) + + return output_paths, compressed_paths, metadata_path + +def pipeline_multi_wrapper( + filepaths, + out_file_stems=None, + fast_mode=False, + force_overwrite=False, + worker_position=None, + quiet=False, + tqdm_lock=None +): + """Fault-tolerant wrapper for multiple inputs. + + Args: + filepaths (_type_): _description_ + out_file_stems (_type_, optional): _description_. Defaults to None. + fast_mode (bool, optional): _description_. Defaults to False. + force_overwrite (bool, optional): _description_. Defaults to False. + + Returns: + _type_: _description_ + """ + + if out_file_stems is not None: + assert len(filepaths) == len(out_file_stems), 'Input filepaths and out_file_stems have different length.' + else: + out_file_stems = [None]*len(filepaths) + + outputs = {'output_paths': [], 'compressed_paths': [], 'metadata_paths': []} + # print(filepaths, out_file_stems) + if tqdm_lock is not None: + tqdm.set_lock(tqdm_lock) + for in_file, out_stem in tqdm(zip(filepaths, out_file_stems), + desc='Processing, worker {}'.format(worker_position), + position=worker_position, + total=len(filepaths), + leave=True): + # try: + output_paths, compressed_paths, metadata_path = pipeline( + in_file, + out_file_stem=out_stem, + fast_mode=fast_mode, + force_overwrite=force_overwrite, + quiet=quiet) + outputs['output_paths'].extend(output_paths) + outputs['compressed_paths'].extend(compressed_paths) + outputs['metadata_paths'].append(metadata_path) + # except: + # print('WARNING: Pipeline failed on input file {}'.format(in_file)) + + return tuple(outputs.values()) + +def parallel_pipeline( + in_file_chunks, + out_stem_chunks=None, + fast_mode=False, + force_overwrite=False, + num_workers=0, + threaded=False, + quiet=False, + desc=None, +): + + if out_stem_chunks is None: + out_stem_chunks = [None]*len(in_file_chunks) + + if len(in_file_chunks) == 0: + return None + else: + assert len(in_file_chunks) == len(out_stem_chunks), 'in_file_chunks and out_stem_chunks must have the same length.' + + if threaded: + executor_cls = concurrent.futures.ThreadPoolExecutor + else: + executor_cls = concurrent.futures.ProcessPoolExecutor + + num_workers = min(len(in_file_chunks), num_workers) + + outputs = {'output_paths': [], 'compressed_paths': [], 'metadata_paths': []} + + lock_manager = Manager() + tqdm_lock = lock_manager.Lock() + + with tqdm(total=len(in_file_chunks), disable=quiet, desc=desc) as progress: + with executor_cls(max_workers=num_workers) as executor: - return output_paths, metadata_path + futures = [executor.submit(pipeline_multi_wrapper, + filepaths=file_chunk, + out_file_stems=out_stem_chunk, + fast_mode=fast_mode, + force_overwrite=force_overwrite, + worker_position=index % num_workers, + quiet=quiet, + tqdm_lock=tqdm_lock) + for index, (file_chunk, out_stem_chunk) in enumerate(zip(in_file_chunks, out_stem_chunks))] + for future in concurrent.futures.as_completed(futures): + output_paths, compressed_paths, metadata_path = future.result() + outputs['output_paths'].extend(output_paths) + outputs['compressed_paths'].extend(compressed_paths) + outputs['metadata_paths'].extend(metadata_path) + progress.update(1) + + return tuple(outputs.values()) def batch( filepaths, @@ -138,7 +251,7 @@ def batch( # Run tiling batch = {} for filepath in filepaths: - _, _, metadata = spectrogram.compute(filepath) + _, _, _, metadata = spectrogram.compute(filepath) batch[filepath] = metadata raise NotImplementedError @@ -152,6 +265,7 @@ def example(): TEST_WAV_HASH = '391efce5433d1057caddb4ce07b9712c523d6a815e4ee9e64b62973569982925' # NOQA wav_filepath = join(PWD, 'examples', 'example1.wav') + # wav_filepath = join(PWD, 'examples', 'extras', '3517_NE_20220622_220344_814.wav') if not exists(wav_filepath): wav_filepath = pooch.retrieve( @@ -166,7 +280,7 @@ def example(): import time start_time = time.time() - results = pipeline(wav_filepath, fast_mode=False) + results = pipeline(wav_filepath, fast_mode=False, force_overwrite=True) stop_time = time.time() print('Example pipeline completed in {} seconds.'.format(stop_time - start_time)) From 57ce5a137e1f91e184f3befaa8bc89bd892d91bd Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Mon, 26 Jan 2026 19:08:56 -0500 Subject: [PATCH 09/33] Added features to preprocess function: use parallel processes, output directory structure to mirror input structure in user specified folder --- batbot/batbot_cli.py | 188 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 155 insertions(+), 33 deletions(-) diff --git a/batbot/batbot_cli.py b/batbot/batbot_cli.py index d0308e6..0b45bee 100755 --- a/batbot/batbot_cli.py +++ b/batbot/batbot_cli.py @@ -4,10 +4,12 @@ """ from glob import glob import json -from os.path import exists +from os.path import exists, commonpath, join, relpath, split, splitext, basename, isdir, isfile +from os import makedirs, getcwd import warnings import click +import numpy as np import batbot from batbot import log @@ -15,6 +17,9 @@ from tqdm import tqdm +import warnings +warnings.filterwarnings("error") + def pipeline_filepath_validator(ctx, param, value): if not exists(value): log.error(f'Input filepath does not exist: {value}') @@ -108,59 +113,176 @@ def pipeline( nargs=-1, type=str, ) -# @click.option( -# '--output-dir', -# help='Processed file output directory. Defaults to current working directory.', -# default='.', -# type=str, -# ) @click.option( - '--metadata', '-m', - help='Use a much slower version of the pipeline which increases spectogram compression quality and outputs additional bat call metadata.', + '--output-dir', '-o', + help='Processed file root output directory. Outputs will attempt to mirror input file directory structure if given multiple inputs (unless --no-file-structure flag is given). Defaults to current working directory.', + nargs=1, + default='.', + type=str, +) +@click.option( + '--process-metadata', '-m', + help='Use a slower version of the pipeline which increases spectogram compression quality and also outputs bat call metadata.', is_flag=True, ) +@click.option( + '--force-overwrite', '-f', + help='Force overwriting of compressed spectrogram and other output files.', + is_flag=True, +) +@click.option( + '--num-workers', '-n', + help='Number of parallel workers to use. Set to zero for serial computation only.', + nargs=1, + default=0, + type=int, +) @click.option( '--output-json', help='Path to output JSON (if unspecified, output file locations are printed to screen)', default=None, type=str, ) -def preprocess(filepaths, metadata, output_json): +@click.option( + '--no-file-structure', + help='(Not recommended) Turn off input file directory structure mirroring. All outputs will be written directly into the provided output dir. WARNING: If multiple input files have the same filename, outputs will overwrite!', + is_flag=True, +) +def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_workers, output_json, no_file_structure): """Generate compressed spectrogram images for wav files into the current working directory. - Takes one or more space separated arguments of filepaths to process. - Filepaths can use wildcards ** for folders and/or * within filenames (if ** wildcard is used, - will recursively search through all subfolders). + Takes one or more space separated arguments of filepaths to process. If given a directory name, + will recursively search through the directory and all subfolders to find all contained *.wav files. + Alternatively, the argument can be given as a string using wildcards ** for folders and/or * + (if ** wildcard is used, will recursively search through all subfolders). + Examples: + batbot preprocess ../data -o ./tmp + batbot preprocess "../data/**/*.wav" + batbot preprocess ../data -o ./tmp -n 32 + batbot preprocess ../data -o ./tmp -n 32 -fm """ - all_filepaths = [] - for file in filepaths: - all_filepaths.extend(glob(file, recursive=True)) + in_filepaths = [] + for input in filepaths: + if isdir(input): + in_filepaths.extend(glob(join(input,'**/*.wav'), recursive=True)) + elif isfile(input): + in_filepaths.append(input) + else: + in_filepaths.extend(glob(input, recursive=True)) # remove any repeats - all_filepaths = sorted(list(set(all_filepaths))) + in_filepaths = sorted(list(set(in_filepaths))) - if len(all_filepaths) == 0: + if len(in_filepaths) == 0: print('Found no files given filepaths input {}'.format(filepaths)) return - print('Running preprocessing on {} located files'.format(len(all_filepaths))) - print('\tFast processing mode {}'.format('OFF' if metadata else 'ON')) - print('\tName of first file to process: {}'.format(all_filepaths[0])) - if len(all_filepaths) > 2: - print('\tName of last file to process: {}'.format(all_filepaths[-1])) + # set up output paths for each input path + root_inpath = commonpath(in_filepaths) + root_outpath = '.' if output_dir is None else output_dir + makedirs(root_outpath, exist_ok=True) + if no_file_structure: + out_filepath_stems = [join(root_outpath, splitext(x)[0]) for x in in_filepaths] + else: + out_filepath_stems = [splitext(join(root_outpath, relpath(x, root_inpath)))[0] for x in in_filepaths] + new_dirs = [split(x)[0] for x in out_filepath_stems] + for new_dir in set(new_dirs): + makedirs(new_dir, exist_ok=True) + + # look for existing output files and remove from the set + in_filepaths = np.array(in_filepaths) + out_filepath_stems = np.array(out_filepath_stems) + if not force_overwrite: + idx_remove = np.full((len(in_filepaths),), False) + for ii, out_file_stem in enumerate(out_filepath_stems): + test_file = '{}.*'.format(out_file_stem) + test_glob = glob(test_file) + if len(test_glob) > 0: + idx_remove[ii] = True + in_filepaths = in_filepaths[np.invert(idx_remove)] + out_filepath_stems = out_filepath_stems[np.invert(idx_remove)] + n_skipped = sum(idx_remove) + if len(in_filepaths) == 0: + print('Found no unprocessed files given filepaths input {} and output directory "{}" after skipping {} files'.format(filepaths, root_outpath, n_skipped)) + print('If desired, use --force-overwrite flag to overwrite existing processed data') + return + + print('Running preprocessing on {} located unprocessed files'.format(len(in_filepaths))) + print('\tFast processing mode {}'.format('OFF' if process_metadata else 'ON')) + if process_metadata: + print('\t\tFull bat call metadata will be produced') + print('\tForce output overwrite {}'.format('ON' if force_overwrite else 'OFF')) + if not force_overwrite: + print('\t\tSkipped {} files with already preprocessed outputs'.format(n_skipped)) + print('\tNum parallel workers: {}'.format(num_workers)) + if no_file_structure: + print('\tFlattening output file structure') + print('\tCurrent working dir: {}'.format(getcwd())) + print('\tOutput root dir: {}'.format(output_dir)) + print('\tFirst input file -> output files: {} -> {}.*'.format(in_filepaths[0], out_filepath_stems[0])) + if len(in_filepaths) > 2: + print('\tLast input file -> output files: {} -> {}.*'.format(in_filepaths[-1], out_filepath_stems[-1])) - data = {'output_path':[], 'metadata_paths':[]} - for file in tqdm(all_filepaths, desc='Preprocessing files', total=len(all_filepaths)): - try: - output_paths, metadata_path = batbot.pipeline(file, fast_mode=(not metadata)) #, extra_arg=True) + # Begin execution loop. + data = {'output_path':[], 'compressed_path':[], 'metadata_path':[]} + if num_workers is None or num_workers == 0: + zipped = np.stack((in_filepaths, out_filepath_stems), axis=-1) + np.random.shuffle(zipped) + assert all([x in zipped[:,0] and y in zipped[:,1] for x, y in zip(in_filepaths, out_filepath_stems)]) + in_filepaths, out_filepath_stems = zipped.T + assert all([basename(y) in basename(x) for x, y in zip(in_filepaths, out_filepath_stems)]) + + # Serial execution. + for file, out_stem in tqdm(zip(in_filepaths, out_filepath_stems), desc='Preprocessing files', total=len(in_filepaths)): + # try: + output_paths, compressed_paths, metadata_path = batbot.pipeline( + file, + out_file_stem=out_stem, + fast_mode=(not process_metadata), + force_overwrite=force_overwrite, + quiet=True, + ) data['output_path'].extend(output_paths) - if metadata: - data['metadata_path'].extend(metadata_path) - except: - warnings.warn('WARNING: Pipeline failed for file {}'.format(file)) + data['compressed_path'].extend(compressed_paths) + if process_metadata: + data['metadata_path'].append(metadata_path) + # except: + # warnings.warn('WARNING: Pipeline failed for file {}'.format(file)) + # raise + else: + # Parallel execution. + # shuffle input and output paths + zipped = np.stack((in_filepaths, out_filepath_stems), axis=-1) + np.random.shuffle(zipped) + assert all([x in zipped[:,0] and y in zipped[:,1] for x, y in zip(in_filepaths, out_filepath_stems)]) + in_filepaths, out_filepath_stems = zipped.T + assert all([basename(y) in basename(x) for x, y in zip(in_filepaths, out_filepath_stems)]) + + # make num_workers chunks + in_file_chunks = np.array_split(in_filepaths, num_workers) + out_stem_chunks = np.array_split(out_filepath_stems, num_workers) + + # send to parallel function + output_paths, compressed_paths, metadata_path = batbot.parallel_pipeline( + in_file_chunks=in_file_chunks, + out_stem_chunks=out_stem_chunks, + fast_mode=(not process_metadata), + force_overwrite=force_overwrite, + num_workers=num_workers, + threaded=False, + quiet=True, + desc='Preprocessing chunks of files with {} workers'.format(num_workers), + ) + data['output_path'].extend(output_paths) + data['compressed_path'].extend(compressed_paths) + if process_metadata: + data['metadata_path'].append(metadata_path) if output_json is None: - print('Processed output paths:') + print('') + print('Full spectrogram output paths:') print(data['output_path']) - if metadata: + print('Compressed spectrogram output paths:') + print(data['compressed_path']) + if process_metadata: print('Processed metadata paths:') print(data['metadata_path']) else: From cd21a41bc497c7326bacd4b116d7112df9eda366 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Mon, 26 Jan 2026 19:15:07 -0500 Subject: [PATCH 10/33] Multiple tweaks and fixes: Disabled line_profiler, image statistics now ignore any high frequencies without data, optionally disable tqdm, added padding in contour finding, handling empty echo masks, optional force overwrite, output filestem specification, reduced some thresholds to handle mostly flat bat calls, added fast mode metadata, handling peak amplitude location movement in processing chain, fixed some bat call metadata values --- batbot/spectrogram/__init__.py | 166 ++++++++++++++++++++++----------- 1 file changed, 111 insertions(+), 55 deletions(-) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index 7fd4c1d..70aaf09 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -1,10 +1,11 @@ """ """ +from glob import glob import json import os import shutil import warnings -from os.path import basename, exists, join, splitext +from os.path import basename, exists, join, split, splitext import cv2 import librosa @@ -14,13 +15,13 @@ # import networkx as nx import numpy as np import pyastar2d -import scipy.signal # Ensure this is at the top with other imports import tqdm from line_profiler import LineProfiler from scipy import ndimage # from PIL import Image -from scipy.ndimage import gaussian_filter1d +from scipy.ndimage import gaussian_filter1d, median_filter +import scipy.stats # from scipy.ndimage.filters import maximum_filter1d from shapely.geometry import Point @@ -38,8 +39,8 @@ def compute(*args, **kwargs): retval = compute_wrapper(*args, **kwargs) - if not kwargs.get('fast_mode', True): - lp.print_stats() + # if not kwargs.get('fast_mode', True) and not kwargs.get('quiet', True): + # lp.print_stats() return retval @@ -114,6 +115,7 @@ def plot_histogram( ignore_zeros=False, max_val=None, smoothing=128, + min_band_idx=None, csum_threshold=0.95, output_path='.', output_filename='histogram.png', @@ -121,6 +123,9 @@ def plot_histogram( if max_val is None: max_val = int(image.max()) + if min_band_idx is not None: + image = image[min_band_idx:, :] + if ignore_zeros: image = image[image > 0] @@ -256,13 +261,13 @@ def load_stft( # Load WAV file try: - waveform_, sr_ = librosa.load(wav_filepath, sr=None) - duration = len(waveform_) / sr_ + waveform_, orig_sr = librosa.load(wav_filepath, sr=None) + duration = len(waveform_) / orig_sr except Exception as e: raise OSError(f'Error loading file: {e}') # Resample the waveform - waveform = librosa.resample(waveform_, orig_sr=sr_, target_sr=sr) + waveform = librosa.resample(waveform_, orig_sr=orig_sr, target_sr=sr) # Convert the waveform to a (complex) spectrogram stft = librosa.stft( @@ -298,11 +303,11 @@ def load_stft( else: waveplot = generate_waveplot(waveform, stft_db, hop_length=hop_length) - return stft_db, waveplot, sr, bands, duration, min_index, time_vec + return stft_db, waveplot, sr, bands, duration, min_index, time_vec, orig_sr # @lp -def gain_stft(stft_db, gain_db=80.0, autogain_stddev=5.0, fast_mode=False): +def gain_stft(stft_db, gain_db=80.0, autogain_stddev=5.0, fast_mode=False, max_band_idx=None): # Subtract per-frequency median DB med = np.median(stft_db, axis=1).reshape(-1, 1) stft_db -= med @@ -316,7 +321,10 @@ def gain_stft(stft_db, gain_db=80.0, autogain_stddev=5.0, fast_mode=False): if not fast_mode: # Calculate the non-zero median DB and MAD # autogain signal if (median - alpha * deviation) is higher than provided gain - temp = stft_db[stft_db > 0] + if max_band_idx is not None: + temp = stft_db[:max_band_idx+1,:][stft_db[:max_band_idx+1,:] > 0] + else: + temp = stft_db[stft_db > 0] med_db = np.median(temp) std_db = scipy.stats.median_abs_deviation(temp, axis=None, scale='normal') autogain_value = med_db - (autogain_stddev * std_db) @@ -377,9 +385,12 @@ def calculate_window_and_stride( return window, stride -def create_coarse_candidates(stft_db, window, stride, threshold_stddev=3.0): +def create_coarse_candidates(stft_db, window, stride, threshold_stddev=3.0, min_band_idx=None): # Re-calculate the non-zero median DB and MAD (scaled like std) - temp = stft_db[stft_db > 0] + if min_band_idx is not None: + temp = stft_db[min_band_idx:, :][stft_db[min_band_idx:, :] > 0] + else: + temp = stft_db[stft_db > 0] med_db = np.median(temp) std_db = scipy.stats.median_abs_deviation(temp, axis=None, scale='normal') threshold = med_db + (threshold_stddev * std_db) @@ -412,8 +423,10 @@ def filter_candidates_to_ranges( window=16, skew_stddev=2.0, area_percent=0.10, + min_band_idx=None, output_path=None, fast_mode=False, + quiet=False, ): # Filter the candidates based on their distribution skewness stride_ = 2 if not fast_mode else 4 @@ -421,9 +434,12 @@ def filter_candidates_to_ranges( reject_idxs = [] ranges = [] - for index, (idx, start, stop) in tqdm.tqdm(list(enumerate(candidates)), disable=fast_mode): + for index, (idx, start, stop) in tqdm.tqdm(list(enumerate(candidates)), disable=quiet): # Extract the candidate window of the STFT - candidate = stft_db[:, start:stop] + if min_band_idx is not None: + candidate = stft_db[min_band_idx:, start:stop] + else: + candidate = stft_db[:, start:stop] # Create a vertical (frequency) sliding window of Numpy views views = np.lib.stride_tricks.sliding_window_view(candidate, (window, candidate.shape[1]))[ @@ -558,7 +574,7 @@ def calculate_mean_within_stddev_window(values, window): def tighten_ranges( - stft_db, ranges, window, duration, skew_stddev=2.0, min_duration_ms=2.0, output_path='.' + stft_db, ranges, window, duration, skew_stddev=2.0, min_duration_ms=2.0, output_path='.', quiet=False ): minimum_duration = int(np.around(stft_db.shape[1] / (duration * 1e3) * min_duration_ms)) @@ -567,7 +583,7 @@ def tighten_ranges( buffer = int(round(window / stride_ / 2)) ranges_ = [] - for index, (start, stop) in tqdm.tqdm(list(enumerate(ranges))): + for index, (start, stop) in tqdm.tqdm(list(enumerate(ranges)), disable=quiet): # Extract the candidate window of the STFT candidate = stft_db[:, start:stop] @@ -724,7 +740,7 @@ def threshold_contour(segment, index, output_path='.'): def filter_contour(segment, index, med_db=None, std_db=None, kernel=5, output_path='.'): # segment = cv2.erode(segment, np.ones((3, 3), np.uint8), iterations=1) - segment = scipy.signal.medfilt(segment, kernel_size=kernel) + segment = median_filter(segment, size=(kernel,kernel), mode='reflect') if None not in {med_db, std_db}: segment_threshold = med_db - std_db @@ -1093,13 +1109,14 @@ def significant_contour_path( return bandwidth, duration, significant -def scale_pdf_contour(segment, index, output_path='.'): +def scale_pdf_contour(segment, index, min_band_idx=None, output_path='.'): segment = normalize_stft(segment, None, segment.dtype) med_db, std_db, peak_db = plot_histogram( segment, smoothing=512, ignore_zeros=True, csum_threshold=0.9, + min_band_idx=min_band_idx, output_path=output_path, output_filename=f'contour.{index}.00.histogram.png', ) @@ -1197,9 +1214,14 @@ def find_contour_and_peak( # (note that these were computed prior to CDF weighting) threshold = peak_db - threshold_std * peak_db_std + # pad left and right edges to handle signal that extends beyond segment edge + segment_pad = np.pad(segment, ((0,0),(1,1))) contours = measure.find_contours( - segment, level=threshold, fully_connected='high', positive_orientation='high' + segment_pad, level=threshold, fully_connected='high', positive_orientation='high' ) + # remove padding in output contour + for contour in contours: + contour[:,1] -= 1 # Display the image and plot all contours found if output_path: @@ -1280,10 +1302,13 @@ def calculate_harmonic_and_echo_flags( negative_skew = scipy.stats.skew(original[np.logical_and(nonzeros, negative)]) harmonic_skew = scipy.stats.skew(original[np.logical_and(nonzeros, harmonic)]) - negative_skew - echo_skew = ( - scipy.stats.skew(original[np.logical_and(np.logical_and(nonzeros, echo), ~harmonic)]) - - negative_skew - ) + if echo.any(): + echo_skew = ( + scipy.stats.skew(original[np.logical_and(np.logical_and(nonzeros, echo), ~harmonic)]) + - negative_skew + ) + else: + echo_skew = -np.inf skew_thresh = np.abs(negative_skew * 0.1) harmonic_flag = harmonic_skew >= skew_thresh @@ -1327,10 +1352,12 @@ def calculate_harmonic_and_echo_flags( # @lp def compute_wrapper( wav_filepath, + out_file_stem=None, + fast_mode=False, + force_overwrite=False, + quiet=False, annotations=None, - output_folder='.', bitdepth=16, - fast_mode=False, debug=False, **kwargs, ): @@ -1354,24 +1381,38 @@ def compute_wrapper( - tuple of spectrogram's (min, max) frequency - list of spectrogram filepaths, split by 50k horizontal pixels """ - base = splitext(basename(wav_filepath))[0] + if not force_overwrite: + test_file = '{}.*'.format(out_file_stem) + test_glob = glob(test_file) + if len(test_glob) > 0: + print('NOTE: Found existing file(s) at {} with force_overwrite=False. Skipping file.'.format(test_file)) + return [], [], [], {} if fast_mode: bitdepth = 8 + quiet=True assert bitdepth in [8, 16] dtype = np.uint8 if bitdepth == 8 else np.uint16 chunksize = int(50e3) - debug_path = get_debug_path(output_folder, wav_filepath, enabled=debug) + debug_path = get_debug_path(split(out_file_stem)[0], wav_filepath, enabled=debug) # Load the spectrogram from a WAV file on disk - stft_db, waveplot, sr, bands, duration, freq_offset, time_vec = load_stft( - wav_filepath, fast_mode=fast_mode - ) + # warnings.filterwarnings('ignore') + with warnings.catch_warnings(): + warnings.simplefilter('ignore', category=DeprecationWarning) + # ignore warning due to aifc deprecation + stft_db, waveplot, sr, bands, duration, freq_offset, time_vec, orig_sr = load_stft( + wav_filepath, fast_mode=fast_mode + ) + # Estimate maximum frequency band containing data + # Only data up to this maximum band should be used when computing statistics + max_band_idx = min((int(np.where(bands < orig_sr / 2.02)[0][-1]), len(bands)-1)) + # warnings.filterwarnings('error') # Apply a dynamic range to a fixed dB range - stft_db = gain_stft(stft_db, fast_mode=fast_mode) + stft_db = gain_stft(stft_db, fast_mode=fast_mode, max_band_idx=max_band_idx) # Bin the floating point data to X-bit integers (X=8 or X=16) stft_db = normalize_stft(stft_db, None, dtype) @@ -1383,6 +1424,7 @@ def compute_wrapper( y_step_freq = float(bands[0] - bands[1]) x_step_ms = float(1e3 * (time_vec[1] - time_vec[0])) bands = np.around(bands).astype(np.int32).tolist() + min_band_idx = len(bands) - max_band_idx - 1 # # Save the spectrogram image to disk # cv2.imwrite('debug.tif', stft_db, [cv2.IMWRITE_TIFF_COMPRESSION, 1]) @@ -1390,7 +1432,7 @@ def compute_wrapper( if not fast_mode: # Plot the histogram, ignoring any non-zero values (will no-op if output_path is None) global_med_db, global_std_db, global_peak_db = plot_histogram( - stft_db, ignore_zeros=True, smoothing=512, output_path=debug_path + stft_db, ignore_zeros=True, smoothing=512, min_band_idx=min_band_idx, output_path=debug_path ) # Estimate a global threshold for finding the edges of bat call contours global_threshold_std = 2.0 @@ -1404,7 +1446,7 @@ def compute_wrapper( # Fast mode uses a relatively small window and lots of overlap, since it skips range tightening step strides_per_window = 3 if not fast_mode else 16 window_size_ms = 12 if not fast_mode else 3 - threshold_stddev = 3.0 if not fast_mode else 4.5 + threshold_stddev = 3.0 if not fast_mode else 4.0 window, stride = calculate_window_and_stride( stft_db, duration, @@ -1413,7 +1455,7 @@ def compute_wrapper( time_vec=time_vec, ) candidates, candidate_max_dbs = create_coarse_candidates( - stft_db, window, stride, threshold_stddev=threshold_stddev + stft_db, window, stride, threshold_stddev=threshold_stddev, min_band_idx=min_band_idx, ) if fast_mode: @@ -1427,8 +1469,9 @@ def compute_wrapper( candidate_max_dbs = [] # Filter all candidates to the ranges that have a substantial right-side skew + ranges, reject_idxs = filter_candidates_to_ranges( - stft_db, candidates, output_path=debug_path, fast_mode=fast_mode + stft_db, candidates, area_percent=0.01, min_band_idx=min_band_idx, output_path=debug_path, fast_mode=fast_mode, quiet=quiet ) # Add in user-specified annotations to ranges @@ -1445,20 +1488,31 @@ def compute_wrapper( plot_chirp_candidates(stft_db, candidate_max_dbs, ranges, reject_idxs, output_path=debug_path) if fast_mode: - # Apply reduced processing without segment refinement or metadata calculation + # Apply reduced processing without segment refinement or per-call metadata calculation segments = {'stft_db': []} # Remove a fraction of the window length when not doing call segmentation crop_length_l = max(0, int(round(0.15 * window - 1))) crop_length_r = max(0, int(round(0.45 * window - 1))) + metas = [] for start, stop in ranges: segments['stft_db'].append(stft_db[:, start + crop_length_l : stop - crop_length_r]) - metas = {} + # Add basic metadata + metadata = { + 'segment start.ms': (start + crop_length_l) * x_step_ms, + 'segment end.ms': (stop - crop_length_r) * x_step_ms, + 'segment duration.ms': (stop - crop_length_r - start - crop_length_l) * x_step_ms, + } + # Normalize values + for key, value in list(metadata.items()): + if key.endswith('.ms'): + metadata[key] = round(float(value), 3) + metas.append(metadata) else: # Tighten the ranges by looking for substantial right-side skew (use stride for a smaller sampling window) - ranges = tighten_ranges(stft_db, ranges, stride, duration, output_path=debug_path) + ranges = tighten_ranges(stft_db, ranges, stride, duration, output_path=debug_path, quiet=quiet) # Extract chirp metrics and metadata segments = { @@ -1468,18 +1522,15 @@ def compute_wrapper( 'canvas': [], } metas = [] - for index, (start, stop) in tqdm.tqdm(list(enumerate(ranges))): + for index, (start, stop) in tqdm.tqdm(list(enumerate(ranges)), disable=quiet): segment = stft_db[:, start:stop] # Step 0.1 - Debugging setup and find peak amplitude (will return None if disabled) canvas = create_contour_debug_canvas(segment, index, output_path=debug_path) - # Step 0.2 - Find the location(s) of peak amplitude - max_locations = find_max_locations(segment) - # Step 1 - Scale with PDF segment, peak_db, peak_db_std = scale_pdf_contour( - segment, index, output_path=debug_path + segment, index, min_band_idx=min_band_idx, output_path=debug_path ) if None in {peak_db, peak_db_std}: continue @@ -1493,6 +1544,9 @@ def compute_wrapper( # Step 4 - Normalize contour segment = normalize_contour(segment, index, output_path=debug_path) + # Step 4.1 - Find the location(s) of peak amplitude + max_locations = find_max_locations(segment) + # # Step 5 (OLD) - Threshold contour # segment, med_db, std_db, peak_db = threshold_contour(segment, index, output_path=debug_path) @@ -1528,10 +1582,10 @@ def compute_wrapper( original, index, segmentmask, harmonic, echo, canvas, output_path=debug_path ) - # Remove harmonic and echo from segmentation - segment = remove_harmonic_and_echo( - segment, index, harmonic, echo, global_threshold, output_path=debug_path - ) + # # Remove harmonic and echo from segmentation ### TODO: make optional + # segment = remove_harmonic_and_echo( + # segment, index, harmonic, echo, global_threshold, output_path=debug_path + # ) # Step 8 - Calculate the A* cost grid and bat call start/end points costs, grid, call_begin, call_end, boundary = calculate_astar_grid_and_endpoints( @@ -1540,8 +1594,10 @@ def compute_wrapper( top, bottom, left, right = boundary # Skip chirp if the extracted path covers a small duration or bandwidth + min_bandwidth_khz = 1e3 + min_duration_ms = 2.0 bandwidth, duration_, significant = significant_contour_path( - call_begin, call_end, y_step_freq, x_step_ms + call_begin, call_end, y_step_freq, x_step_ms, min_bandwidth_khz=min_bandwidth_khz, min_duration_ms=min_duration_ms ) if not significant: continue @@ -1572,9 +1628,9 @@ def compute_wrapper( round(255.0 * (segment_threshold / np.iinfo(stft_db.dtype).max)) ), 'peak f.ms': (start + peak[1]) * x_step_ms, - 'fc.ms': (start + bands[fc[1]]) * x_step_ms, - 'hi fc:knee.ms': (start + bands[knee[1]]) * x_step_ms, - 'lo fc:heel.ms': (start + bands[heel[1]]) * x_step_ms, + 'fc.ms': (start + fc[1]) * x_step_ms, + 'hi fc:knee.ms': (start + knee[1]) * x_step_ms, + 'lo fc:heel.ms': (start + heel[1]) * x_step_ms, 'bandwidth.hz': bandwidth, 'hi f.hz': bands[top], 'lo f.hz': bands[bottom], @@ -1724,7 +1780,7 @@ def compute_wrapper( for index, chunk in enumerate(chunks): if chunk.shape[1] == 0: continue - output_path = join(output_folder, f'{base}.{index + 1:02d}of{total:02d}.{tag}') + output_path = f'{out_file_stem}.{index + 1:02d}of{total:02d}.{tag}' cv2.imwrite(output_path, chunk, [cv2.IMWRITE_JPEG_QUALITY, 80]) accumulator.append(output_path) @@ -1762,8 +1818,8 @@ def compute_wrapper( }, ) - metadata_path = join(output_folder, f'{base}.metadata.json') + metadata_path = f'{out_file_stem}.metadata.json' with open(metadata_path, 'w') as metafile: json.dump(metadata, metafile, indent=4) - return output_paths, metadata_path, metadata + return output_paths, compressed_paths, metadata_path, metadata From b0987454a22650e299f1d10a670642a263f05d1b Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Tue, 27 Jan 2026 12:07:59 -0500 Subject: [PATCH 11/33] Various cleanup, adding segment start and end metadata --- batbot/batbot_cli.py | 3 +- batbot/spectrogram/__init__.py | 70 +++++++++++++++++----------------- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/batbot/batbot_cli.py b/batbot/batbot_cli.py index 0b45bee..62c6d27 100755 --- a/batbot/batbot_cli.py +++ b/batbot/batbot_cli.py @@ -17,8 +17,7 @@ from tqdm import tqdm -import warnings -warnings.filterwarnings("error") +# warnings.filterwarnings("error") def pipeline_filepath_validator(ctx, param, value): if not exists(value): diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index 70aaf09..90bc83c 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -5,11 +5,10 @@ import os import shutil import warnings -from os.path import basename, exists, join, split, splitext +from os.path import basename, exists, join, split import cv2 import librosa -import librosa.display import matplotlib.pyplot as plt # import networkx as nx @@ -23,7 +22,6 @@ from scipy.ndimage import gaussian_filter1d, median_filter import scipy.stats -# from scipy.ndimage.filters import maximum_filter1d from shapely.geometry import Point from shapely.geometry.polygon import Polygon from skimage import draw, measure @@ -1358,6 +1356,7 @@ def compute_wrapper( quiet=False, annotations=None, bitdepth=16, + mask_secondary_effects=False, debug=False, **kwargs, ): @@ -1399,7 +1398,6 @@ def compute_wrapper( debug_path = get_debug_path(split(out_file_stem)[0], wav_filepath, enabled=debug) # Load the spectrogram from a WAV file on disk - # warnings.filterwarnings('ignore') with warnings.catch_warnings(): warnings.simplefilter('ignore', category=DeprecationWarning) # ignore warning due to aifc deprecation @@ -1409,7 +1407,6 @@ def compute_wrapper( # Estimate maximum frequency band containing data # Only data up to this maximum band should be used when computing statistics max_band_idx = min((int(np.where(bands < orig_sr / 2.02)[0][-1]), len(bands)-1)) - # warnings.filterwarnings('error') # Apply a dynamic range to a fixed dB range stft_db = gain_stft(stft_db, fast_mode=fast_mode, max_band_idx=max_band_idx) @@ -1426,9 +1423,6 @@ def compute_wrapper( bands = np.around(bands).astype(np.int32).tolist() min_band_idx = len(bands) - max_band_idx - 1 - # # Save the spectrogram image to disk - # cv2.imwrite('debug.tif', stft_db, [cv2.IMWRITE_TIFF_COMPRESSION, 1]) - if not fast_mode: # Plot the histogram, ignoring any non-zero values (will no-op if output_path is None) global_med_db, global_std_db, global_peak_db = plot_histogram( @@ -1496,7 +1490,13 @@ def compute_wrapper( crop_length_r = max(0, int(round(0.45 * window - 1))) metas = [] for start, stop in ranges: - segments['stft_db'].append(stft_db[:, start + crop_length_l : stop - crop_length_r]) + if start > 0 and stop < stft_db.shape[1]: + segments['stft_db'].append(stft_db[:, start + crop_length_l : stop - crop_length_r]) + elif start > 0: + # handle cases where candidate butts up against data edges + segments['stft_db'].append(stft_db[:, start + crop_length_l : stop]) + else: + segments['stft_db'].append(stft_db[:, start : stop - crop_length_r]) # Add basic metadata metadata = { 'segment start.ms': (start + crop_length_l) * x_step_ms, @@ -1547,9 +1547,6 @@ def compute_wrapper( # Step 4.1 - Find the location(s) of peak amplitude max_locations = find_max_locations(segment) - # # Step 5 (OLD) - Threshold contour - # segment, med_db, std_db, peak_db = threshold_contour(segment, index, output_path=debug_path) - # Step 5 - Find primary contour that contains max amplitude # (To use a local instead of global threshold, remove the threshold argument here) segmentmask, peak, segment_threshold = find_contour_and_peak( @@ -1568,11 +1565,6 @@ def compute_wrapper( # Step 6 - Create final segmentmask segmentmask = refine_segmentmask(segmentmask, index, output_path=debug_path) - # # Step 6 (OLD) - Find the contour with the (most) max amplitude location(s) - # valid, segmentmask, peak = find_contour_connected_components(segment, index, max_locations, output_path=debug_path) - # # Step 6 (OLD) - Refine contour by removing any harmonic or echo - # segmentmask, peak = refine_contour(segment_, index, max_locations, segmentmask, peak, output_path=debug_path) - # Step 7 - Calculate the first order harmonic and echo region harmonic = find_harmonic(segmentmask, index, freq_offset, output_path=debug_path) echo = find_echo(segmentmask, index, output_path=debug_path) @@ -1582,10 +1574,11 @@ def compute_wrapper( original, index, segmentmask, harmonic, echo, canvas, output_path=debug_path ) - # # Remove harmonic and echo from segmentation ### TODO: make optional - # segment = remove_harmonic_and_echo( - # segment, index, harmonic, echo, global_threshold, output_path=debug_path - # ) + # Remove harmonic and echo from segmentation + if mask_secondary_effects: + segment = remove_harmonic_and_echo( + segment, index, harmonic, echo, global_threshold, output_path=debug_path + ) # Step 8 - Calculate the A* cost grid and bat call start/end points costs, grid, call_begin, call_end, boundary = calculate_astar_grid_and_endpoints( @@ -1621,9 +1614,9 @@ def compute_wrapper( ) for y, x in path_smoothed ], - 'start.ms': (start + left) * x_step_ms, - 'end.ms': (start + right) * x_step_ms, - 'duration.ms': (right - left) * x_step_ms, + 'contour start.ms': (start + left) * x_step_ms, + 'contour end.ms': (start + right) * x_step_ms, + 'contour duration.ms': (right - left) * x_step_ms, 'threshold.amp': int( round(255.0 * (segment_threshold / np.iinfo(stft_db.dtype).max)) ), @@ -1649,6 +1642,23 @@ def compute_wrapper( } metadata.update(slopes) + # Trim segment around the bat call with a small buffer + buffer_ms = 1.0 + buffer_pix = int(round(buffer_ms / x_step_ms)) + trim_begin = max(0, min(segment.shape[1], call_begin[1] - buffer_pix)) + trim_end = max(0, min(segment.shape[1], call_end[1] + buffer_pix)) + + segments['stft_db'].append(stft_db[:, start + trim_begin : start + trim_end]) + segments['waveplot'].append(waveplot[:, start + trim_begin : start + trim_end]) + segments['costs'].append(costs[:, trim_begin:trim_end]) + if debug_path: + segments['canvas'].append(canvas[:, trim_begin:trim_end]) + + # Update metadata with segment start and stop + start_stop = {'segment start.ms': (start + trim_begin) * x_step_ms, + 'segment end.ms': (start + trim_end) * x_step_ms} + metadata.update(start_stop) + # Normalize values for key, value in list(metadata.items()): if value is None: @@ -1674,18 +1684,6 @@ def compute_wrapper( metas.append(metadata) - # Trim segment around the bat call with a small buffer - buffer_ms = 1.0 - buffer_pix = int(round(buffer_ms / x_step_ms)) - trim_begin = max(0, min(segment.shape[1], call_begin[1] - buffer_pix)) - trim_end = max(0, min(segment.shape[1], call_end[1] + buffer_pix)) - - segments['stft_db'].append(stft_db[:, start + trim_begin : start + trim_end]) - segments['waveplot'].append(waveplot[:, start + trim_begin : start + trim_end]) - segments['costs'].append(costs[:, trim_begin:trim_end]) - if debug_path: - segments['canvas'].append(canvas[:, trim_begin:trim_end]) - # Concatenate extracted, trimmed segments and other matrices for key in list(segments.keys()): value = segments[key] From 1af62cedbbc80c4df8604a4118f5c17da9b3b85a Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Tue, 27 Jan 2026 12:13:29 -0500 Subject: [PATCH 12/33] Add segment duration metadata --- batbot/spectrogram/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index 90bc83c..1917bea 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -1656,7 +1656,8 @@ def compute_wrapper( # Update metadata with segment start and stop start_stop = {'segment start.ms': (start + trim_begin) * x_step_ms, - 'segment end.ms': (start + trim_end) * x_step_ms} + 'segment end.ms': (start + trim_end) * x_step_ms, + 'segment duration.ms': (trim_end - trim_begin) * x_step_ms} metadata.update(start_stop) # Normalize values From 9ba1fa7e503d1a7b106bfca65c360091f8311b31 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Tue, 27 Jan 2026 15:26:37 -0500 Subject: [PATCH 13/33] Added preprocess dry run and cleanup modes, added indents to json outputs --- batbot/batbot_cli.py | 115 +++++++++++++++++++++++++++++++++---------- 1 file changed, 88 insertions(+), 27 deletions(-) diff --git a/batbot/batbot_cli.py b/batbot/batbot_cli.py index 62c6d27..f93edd1 100755 --- a/batbot/batbot_cli.py +++ b/batbot/batbot_cli.py @@ -5,7 +5,7 @@ from glob import glob import json from os.path import exists, commonpath, join, relpath, split, splitext, basename, isdir, isfile -from os import makedirs, getcwd +from os import makedirs, getcwd, remove import warnings import click @@ -102,7 +102,7 @@ def pipeline( log.debug('Outputting results...') if output: with open(output, 'w') as outfile: - json.dump(data, outfile) + json.dump(data, outfile, indent=4) else: print(data) @@ -142,31 +142,43 @@ def pipeline( default=None, type=str, ) +@click.option( + '--dry-run', '-d', + help='List out all the audio files to be loaded and all the anticipated output files. Additionally lists all "extra" files in the output directory that would be deleted if using the --cleanup flag.', + is_flag=True, +) +@click.option( + '--cleanup', + help='For the given input filepaths and --output-dir arguments, delete any extra files that would not have been created by the batbot preprocess. Acts as if --force-overwrite flag is given. WARNING: This will delete files, recommend running with the --dry-run flag first and carefully examining the output!', + is_flag=True, +) @click.option( '--no-file-structure', help='(Not recommended) Turn off input file directory structure mirroring. All outputs will be written directly into the provided output dir. WARNING: If multiple input files have the same filename, outputs will overwrite!', is_flag=True, ) -def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_workers, output_json, no_file_structure): +def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_workers, output_json, dry_run, cleanup, no_file_structure): """Generate compressed spectrogram images for wav files into the current working directory. Takes one or more space separated arguments of filepaths to process. If given a directory name, will recursively search through the directory and all subfolders to find all contained *.wav files. - Alternatively, the argument can be given as a string using wildcards ** for folders and/or * + Alternatively, the argument can be given as a string using wildcard ** for folders and/or * in filenames (if ** wildcard is used, will recursively search through all subfolders). Examples: batbot preprocess ../data -o ./tmp batbot preprocess "../data/**/*.wav" batbot preprocess ../data -o ./tmp -n 32 batbot preprocess ../data -o ./tmp -n 32 -fm + batbot preprocess ../data -o ./tmp -f --dry-run --output-json dry_run.json + batbot preprocess ../data -o ./tmp --cleanup """ in_filepaths = [] - for input in filepaths: - if isdir(input): - in_filepaths.extend(glob(join(input,'**/*.wav'), recursive=True)) - elif isfile(input): - in_filepaths.append(input) + for file in filepaths: + if isdir(file): + in_filepaths.extend(glob(join(file,'**/*.wav'), recursive=True)) + elif isfile(file): + in_filepaths.append(file) else: - in_filepaths.extend(glob(input, recursive=True)) + in_filepaths.extend(glob(file, recursive=True)) # remove any repeats in_filepaths = sorted(list(set(in_filepaths))) @@ -188,6 +200,9 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor # look for existing output files and remove from the set in_filepaths = np.array(in_filepaths) + if dry_run or cleanup: + # save copy of all outputs before removing already processed data + out_filepath_stems_all = out_filepath_stems.copy() out_filepath_stems = np.array(out_filepath_stems) if not force_overwrite: idx_remove = np.full((len(in_filepaths),), False) @@ -204,7 +219,21 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor print('If desired, use --force-overwrite flag to overwrite existing processed data') return - print('Running preprocessing on {} located unprocessed files'.format(len(in_filepaths))) + if dry_run or cleanup: + # Find all "extra" files that would be deleted in cleanup mode + all_files = set(glob(join(root_outpath, '**/*'), recursive=True)) + for out_stem in out_filepath_stems_all: + out_files = glob('{}.*'.format(out_stem)) + all_files -= set(out_files) + dir_files = [] + # remove directories + for file in all_files: + if isdir(file): + dir_files.append(file) + all_files -= set(dir_files) + extra_files = all_files + + print('Located {} total unprocessed files'.format(len(in_filepaths))) print('\tFast processing mode {}'.format('OFF' if process_metadata else 'ON')) if process_metadata: print('\t\tFull bat call metadata will be produced') @@ -219,6 +248,37 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor print('\tFirst input file -> output files: {} -> {}.*'.format(in_filepaths[0], out_filepath_stems[0])) if len(in_filepaths) > 2: print('\tLast input file -> output files: {} -> {}.*'.format(in_filepaths[-1], out_filepath_stems[-1])) + + if dry_run: + # Print out files to be processed, anticipated outputs, and files that would be deleted in cleanup mode. + print('\nDry run mode active - skipping all processing') + data = {} + data['input file, output file stem'] = [(str(x),'{}.*'.format(y)) for x, y in zip(in_filepaths, out_filepath_stems)] + data['files to be deleted in cleanup'] = list(extra_files) + if output_json is None: + import pprint + pprint.pp(data) + else: + with open(output_json, 'w') as outfile: + json.dump(data, outfile, indent=4) + print('Outputs written to {}'.format(output_json)) + print('Complete.') + return + + if cleanup: + print('\nCleanup mode active - skipping all processing') + if len(extra_files) == 0: + print('No files to delete') + else: + usr_in = input('Found {} files to delete (recommend to see details by running with --dry-run flag). Continue (y/n)? '.format(len(extra_files))) + if usr_in.lower() not in ['y', 'yes']: + print('Aborting cleanup mode.') + return + for file in extra_files: + print('Deleting file: {}'.format(file)) + remove(file) + print('Complete.') + return # Begin execution loop. data = {'output_path':[], 'compressed_path':[], 'metadata_path':[]} @@ -231,20 +291,20 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor # Serial execution. for file, out_stem in tqdm(zip(in_filepaths, out_filepath_stems), desc='Preprocessing files', total=len(in_filepaths)): - # try: - output_paths, compressed_paths, metadata_path = batbot.pipeline( - file, - out_file_stem=out_stem, - fast_mode=(not process_metadata), - force_overwrite=force_overwrite, - quiet=True, - ) - data['output_path'].extend(output_paths) - data['compressed_path'].extend(compressed_paths) - if process_metadata: - data['metadata_path'].append(metadata_path) - # except: - # warnings.warn('WARNING: Pipeline failed for file {}'.format(file)) + try: + output_paths, compressed_paths, metadata_path = batbot.pipeline( + file, + out_file_stem=out_stem, + fast_mode=(not process_metadata), + force_overwrite=force_overwrite, + quiet=True, + ) + data['output_path'].extend(output_paths) + data['compressed_path'].extend(compressed_paths) + if process_metadata: + data['metadata_path'].append(metadata_path) + except: + warnings.warn('WARNING: Pipeline failed for file {}'.format(file)) # raise else: # Parallel execution. @@ -286,7 +346,8 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor print(data['metadata_path']) else: with open(output_json, 'w') as outfile: - json.dump(data, outfile) + json.dump(data, outfile, indent=4) + print('Outputs written to {}'.format(output_json)) print('Complete.') return data @@ -359,7 +420,7 @@ def batch( log.debug('Outputting results...') if output: with open(output, 'w') as outfile: - json.dump(data, outfile) + json.dump(data, outfile, indent=4) else: print(data) From 87f3627f08d5b879365a9651e0fa3f36f2206bbf Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Tue, 27 Jan 2026 15:27:52 -0500 Subject: [PATCH 14/33] Re-enabled fault tolerance in parallel pipeline --- batbot/__init__.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/batbot/__init__.py b/batbot/__init__.py index 56cebfc..bbe8219 100644 --- a/batbot/__init__.py +++ b/batbot/__init__.py @@ -144,18 +144,19 @@ def pipeline_multi_wrapper( position=worker_position, total=len(filepaths), leave=True): - # try: - output_paths, compressed_paths, metadata_path = pipeline( - in_file, - out_file_stem=out_stem, - fast_mode=fast_mode, - force_overwrite=force_overwrite, - quiet=quiet) - outputs['output_paths'].extend(output_paths) - outputs['compressed_paths'].extend(compressed_paths) - outputs['metadata_paths'].append(metadata_path) - # except: - # print('WARNING: Pipeline failed on input file {}'.format(in_file)) + try: + output_paths, compressed_paths, metadata_path = pipeline( + in_file, + out_file_stem=out_stem, + fast_mode=fast_mode, + force_overwrite=force_overwrite, + quiet=quiet) + outputs['output_paths'].extend(output_paths) + outputs['compressed_paths'].extend(compressed_paths) + outputs['metadata_paths'].append(metadata_path) + except: + print('WARNING: Pipeline failed on input file {}'.format(in_file)) + # raise return tuple(outputs.values()) @@ -266,6 +267,7 @@ def example(): wav_filepath = join(PWD, 'examples', 'example1.wav') # wav_filepath = join(PWD, 'examples', 'extras', '3517_NE_20220622_220344_814.wav') + # wav_filepath = join(PWD, 'examples', 'extras', 'p33_g67260_f29842117.wav') if not exists(wav_filepath): wav_filepath = pooch.retrieve( From d502b20e6234f01c69e6fe00a8bafdcc5400026f Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Tue, 27 Jan 2026 22:23:31 -0500 Subject: [PATCH 15/33] Added edge case fixes. Added small buffer to bat call contour --- batbot/spectrogram/__init__.py | 53 ++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index 1917bea..5b3ab40 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -51,7 +51,7 @@ def get_islands(data): def get_slope_islands(slope_flags): - flags = slope_flags.astype(np.uint8) + flags = slope_flags.astype(np.uint16) islands = get_islands(flags) idx = int(np.argmax([val.sum() for val in islands])) islands = [val * (1 if i == idx else 0) for i, val in enumerate(islands)] @@ -443,7 +443,10 @@ def filter_candidates_to_ranges( views = np.lib.stride_tricks.sliding_window_view(candidate, (window, candidate.shape[1]))[ ::stride_, 0 ] - skews = scipy.stats.skew(views, axis=(1, 2)) + with warnings.catch_warnings(): + # handle cases with mono-valued data + warnings.simplefilter('ignore', category=RuntimeWarning) + skews = scipy.stats.skew(views, axis=(1, 2)) # Center and clip the skew values skew_thresh = calculate_mean_within_stddev_window(skews, skew_stddev) @@ -452,7 +455,7 @@ def filter_candidates_to_ranges( skews = normalize_skew(skews, skew_thresh) # Calculate the largest contiguous island of non-zeros - skews = (skews > 0).astype(np.uint8) + skews = (skews > 0).astype(np.uint16) islands = get_islands(skews) area = float(max([val.sum() for val in islands])) area /= len(skews) @@ -555,7 +558,7 @@ def normalize_skew(skews, skew_thresh): with warnings.catch_warnings(): warnings.simplefilter('ignore', category=RuntimeWarning) - skews /= skews.max() + skews /= np.nanmax(skews) skews = np.nan_to_num(skews, nan=0.0, posinf=0.0, neginf=-0.0) @@ -564,8 +567,8 @@ def normalize_skew(skews, skew_thresh): def calculate_mean_within_stddev_window(values, window): # Calculate the average skew within X standard deviations (temperature scaling) - values_mean = np.mean(values) - values_std = np.std(values) + values_mean = np.nanmean(values) + values_std = np.nanstd(values) values_flags = np.abs(values - values_mean) <= (values_std * window) values_mean_windowed = values[values_flags].mean() return values_mean_windowed @@ -585,11 +588,14 @@ def tighten_ranges( # Extract the candidate window of the STFT candidate = stft_db[:, start:stop] - # Create a vertical (frequency) sliding window of Numpy views + # Create a horizontal (time) sliding window of Numpy views views = np.lib.stride_tricks.sliding_window_view(candidate, (candidate.shape[0], window))[ 0, ::stride_ ] - skews = scipy.stats.skew(views, axis=(1, 2)) + with warnings.catch_warnings(): + # handle cases with mono-valued data + warnings.simplefilter('ignore', category=RuntimeWarning) + skews = scipy.stats.skew(views, axis=(1, 2)) # Center and clip the skew values skew_thresh = calculate_mean_within_stddev_window(skews, skew_stddev) @@ -597,7 +603,7 @@ def tighten_ranges( # Calculate the largest contiguous island of non-zeros skew_flags = skews > 0 - skews = skew_flags.astype(np.uint8) + skews = skew_flags.astype(np.uint16) islands = get_islands(skews) islands = [(index + 1) * val for index, val in enumerate(islands)] island = np.hstack(islands) @@ -1212,14 +1218,15 @@ def find_contour_and_peak( # (note that these were computed prior to CDF weighting) threshold = peak_db - threshold_std * peak_db_std - # pad left and right edges to handle signal that extends beyond segment edge - segment_pad = np.pad(segment, ((0,0),(1,1))) + # pad all edges to handle signal that butts up against segment edges + segment_pad = np.pad(segment, ((2,2),(2,2))) contours = measure.find_contours( segment_pad, level=threshold, fully_connected='high', positive_orientation='high' ) # remove padding in output contour for contour in contours: - contour[:,1] -= 1 + contour[:,0] -= 2 + contour[:,1] -= 2 # Display the image and plot all contours found if output_path: @@ -1249,6 +1256,11 @@ def find_contour_and_peak( contour_ = np.vstack((y, x), dtype=contour.dtype).T polygon_ = Polygon(contour).convex_hull + + # Add small buffer to smoothed contour be sure to include maximum value location. + polygon = Polygon(contour).buffer(1.0) + xx, yy = polygon.exterior.coords.xy + contour_ = np.vstack((xx, yy)).T assert idx not in counter counter[idx] = (found, polygon_) @@ -1299,13 +1311,16 @@ def calculate_harmonic_and_echo_flags( write_contour_debug_image(negative_, index, 7, 'negative', output_path=output_path) negative_skew = scipy.stats.skew(original[np.logical_and(nonzeros, negative)]) - harmonic_skew = scipy.stats.skew(original[np.logical_and(nonzeros, harmonic)]) - negative_skew - if echo.any(): - echo_skew = ( - scipy.stats.skew(original[np.logical_and(np.logical_and(nonzeros, echo), ~harmonic)]) - - negative_skew - ) - else: + with warnings.catch_warnings(): + # allow for nan outputs in cases of empty or mono-valued selections + warnings.simplefilter('ignore', category=RuntimeWarning) + selection = np.logical_and(nonzeros, harmonic) + harmonic_skew = scipy.stats.skew(original[selection]) - negative_skew + selection = np.logical_and(np.logical_and(nonzeros, echo), ~harmonic) + echo_skew = scipy.stats.skew(original[selection]) - negative_skew + if np.isnan(harmonic_skew): + harmonic_skew = -np.inf + if np.isnan(echo_skew): echo_skew = -np.inf skew_thresh = np.abs(negative_skew * 0.1) From 3a14eb54a416b1e3825abd563541e1a18645aaaf Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 16:45:57 -0500 Subject: [PATCH 16/33] Fixed STFT low values getting cut off, expanded default dynamic range --- batbot/spectrogram/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index 5b3ab40..581dc68 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -267,12 +267,16 @@ def load_stft( # Resample the waveform waveform = librosa.resample(waveform_, orig_sr=orig_sr, target_sr=sr) + # TODO: signal processing: remove DC offset, time window edges of waveform + # Convert the waveform to a (complex) spectrogram stft = librosa.stft( waveform, n_fft=n_fft, window=window, win_length=win_length, hop_length=hop_length ) # Convert the complex power (amplitude + phase) into amplitude (decibels) - stft_db = librosa.power_to_db(np.abs(stft) ** 2, ref=np.max) + # Do not threshold the data - threshold will be applied later + # stft_db = librosa.power_to_db(np.abs(stft) ** 2, ref=np.max, top_db=np.inf) # OLD method, cuts off lower values + stft_db = 10 * np.log10(np.square(np.abs(stft))) # Retrieve time vector in seconds corresponding to STFT time_vec = librosa.frames_to_time( range(stft_db.shape[1]), sr=sr, hop_length=hop_length, n_fft=n_fft @@ -305,7 +309,7 @@ def load_stft( # @lp -def gain_stft(stft_db, gain_db=80.0, autogain_stddev=5.0, fast_mode=False, max_band_idx=None): +def gain_stft(stft_db, gain_db=120.0, autogain_stddev=5.0, fast_mode=False, max_band_idx=None): # Subtract per-frequency median DB med = np.median(stft_db, axis=1).reshape(-1, 1) stft_db -= med From 9f8db2ebfa6d392c767ce91930694c512d616204 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 17:43:06 -0500 Subject: [PATCH 17/33] STFT outside sample rate support explicitly removed, turned back on autogain in fast mode --- batbot/spectrogram/__init__.py | 41 ++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index 581dc68..862fc7d 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -305,11 +305,18 @@ def load_stft( else: waveplot = generate_waveplot(waveform, stft_db, hop_length=hop_length) - return stft_db, waveplot, sr, bands, duration, min_index, time_vec, orig_sr + # Estimate maximum frequency band containing data based on original sample rate + # Only data up to this maximum band should be used when computing statistics + max_band_idx = min((int(np.where(bands < orig_sr / 2.02)[0][-1]), len(bands)-1)) + # set non-physical noise above the max band to a minimum value + if max_band_idx < len(bands)-1: + stft_db[max_band_idx+1:, :] = np.min(stft_db[:max_band_idx+1, :]) + + return stft_db, waveplot, sr, bands, duration, min_index, time_vec, orig_sr, max_band_idx # @lp -def gain_stft(stft_db, gain_db=120.0, autogain_stddev=5.0, fast_mode=False, max_band_idx=None): +def gain_stft(stft_db, gain_db=120.0, autogain_stddev=5.0, max_band_idx=None): # Subtract per-frequency median DB med = np.median(stft_db, axis=1).reshape(-1, 1) stft_db -= med @@ -320,18 +327,17 @@ def gain_stft(stft_db, gain_db=120.0, autogain_stddev=5.0, fast_mode=False, max_ assert stft_db.max() == 0 stft_db += gain_db - if not fast_mode: - # Calculate the non-zero median DB and MAD - # autogain signal if (median - alpha * deviation) is higher than provided gain - if max_band_idx is not None: - temp = stft_db[:max_band_idx+1,:][stft_db[:max_band_idx+1,:] > 0] - else: - temp = stft_db[stft_db > 0] - med_db = np.median(temp) - std_db = scipy.stats.median_abs_deviation(temp, axis=None, scale='normal') - autogain_value = med_db - (autogain_stddev * std_db) - if autogain_value > 0: - stft_db -= autogain_value + # Calculate the non-zero median DB and MAD + # autogain signal if (median - alpha * deviation) is higher than provided gain + if max_band_idx is not None: + temp = stft_db[:max_band_idx+1,:][stft_db[:max_band_idx+1,:] > 0] + else: + temp = stft_db[stft_db > 0] + med_db = np.median(temp) + std_db = scipy.stats.median_abs_deviation(temp, axis=None, scale='normal') + autogain_value = med_db - (autogain_stddev * std_db) + if autogain_value > 0: + stft_db -= autogain_value # Clip values below zero stft_db = np.clip(stft_db, 0.0, None) @@ -1420,15 +1426,12 @@ def compute_wrapper( with warnings.catch_warnings(): warnings.simplefilter('ignore', category=DeprecationWarning) # ignore warning due to aifc deprecation - stft_db, waveplot, sr, bands, duration, freq_offset, time_vec, orig_sr = load_stft( + stft_db, waveplot, sr, bands, duration, freq_offset, time_vec, orig_sr, max_band_idx = load_stft( wav_filepath, fast_mode=fast_mode ) - # Estimate maximum frequency band containing data - # Only data up to this maximum band should be used when computing statistics - max_band_idx = min((int(np.where(bands < orig_sr / 2.02)[0][-1]), len(bands)-1)) # Apply a dynamic range to a fixed dB range - stft_db = gain_stft(stft_db, fast_mode=fast_mode, max_band_idx=max_band_idx) + stft_db = gain_stft(stft_db, max_band_idx=max_band_idx) # Bin the floating point data to X-bit integers (X=8 or X=16) stft_db = normalize_stft(stft_db, None, dtype) From cf0fbe3adf6a53d90afb7366c006c88607eecd51 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 17:45:24 -0500 Subject: [PATCH 18/33] Adjusted preprocessing terminal prints, now includes list of failed files. Added random seed, removed randomness from serial execution --- batbot/batbot_cli.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/batbot/batbot_cli.py b/batbot/batbot_cli.py index f93edd1..659821b 100755 --- a/batbot/batbot_cli.py +++ b/batbot/batbot_cli.py @@ -149,7 +149,7 @@ def pipeline( ) @click.option( '--cleanup', - help='For the given input filepaths and --output-dir arguments, delete any extra files that would not have been created by the batbot preprocess. Acts as if --force-overwrite flag is given. WARNING: This will delete files, recommend running with the --dry-run flag first and carefully examining the output!', + help='For the given input filepaths and --output-dir arguments, delete any extra files that would not have been created by the batbot preprocess. Skips hidden files starting with ".". Acts as if --force-overwrite flag is given (does not delete existing, preprocessed outputs). WARNING: This will delete files, recommend running with the --dry-run flag first and carefully examining the output!', is_flag=True, ) @click.option( @@ -281,13 +281,8 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor return # Begin execution loop. - data = {'output_path':[], 'compressed_path':[], 'metadata_path':[]} + data = {'output_path':[], 'compressed_path':[], 'metadata_path':[], 'failed_files':[]} if num_workers is None or num_workers == 0: - zipped = np.stack((in_filepaths, out_filepath_stems), axis=-1) - np.random.shuffle(zipped) - assert all([x in zipped[:,0] and y in zipped[:,1] for x, y in zip(in_filepaths, out_filepath_stems)]) - in_filepaths, out_filepath_stems = zipped.T - assert all([basename(y) in basename(x) for x, y in zip(in_filepaths, out_filepath_stems)]) # Serial execution. for file, out_stem in tqdm(zip(in_filepaths, out_filepath_stems), desc='Preprocessing files', total=len(in_filepaths)): @@ -305,11 +300,13 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor data['metadata_path'].append(metadata_path) except: warnings.warn('WARNING: Pipeline failed for file {}'.format(file)) - # raise + data['failed_files'].append(str(file)) + # raise else: # Parallel execution. # shuffle input and output paths zipped = np.stack((in_filepaths, out_filepath_stems), axis=-1) + np.random.seed(0) np.random.shuffle(zipped) assert all([x in zipped[:,0] and y in zipped[:,1] for x, y in zip(in_filepaths, out_filepath_stems)]) in_filepaths, out_filepath_stems = zipped.T @@ -320,7 +317,7 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor out_stem_chunks = np.array_split(out_filepath_stems, num_workers) # send to parallel function - output_paths, compressed_paths, metadata_path = batbot.parallel_pipeline( + output_paths, compressed_paths, metadata_paths, failed_files = batbot.parallel_pipeline( in_file_chunks=in_file_chunks, out_stem_chunks=out_stem_chunks, fast_mode=(not process_metadata), @@ -333,22 +330,25 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor data['output_path'].extend(output_paths) data['compressed_path'].extend(compressed_paths) if process_metadata: - data['metadata_path'].append(metadata_path) + data['metadata_path'].extend(metadata_paths) + data['failed_files'].extend(failed_files) if output_json is None: - print('') - print('Full spectrogram output paths:') - print(data['output_path']) - print('Compressed spectrogram output paths:') - print(data['compressed_path']) + import pprint + print('\nFull spectrogram output paths:') + pprint.pp(sorted(data['output_path'])) + print('\nCompressed spectrogram output paths:') + pprint.pp(sorted(data['compressed_path'])) if process_metadata: - print('Processed metadata paths:') - print(data['metadata_path']) + print('\nProcessed metadata paths:') + pprint.pp(sorted(data['metadata_path'])) + print('\nFiles that failed processing and were skipped:') + pprint.pp(sorted(data['failed_files'])) else: with open(output_json, 'w') as outfile: json.dump(data, outfile, indent=4) print('Outputs written to {}'.format(output_json)) - print('Complete.') + print('\nComplete.') return data From 81cc73998c25d0be4e874cdbb8a9151110b86bf2 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 17:46:41 -0500 Subject: [PATCH 19/33] Added debug argument to pipeline, added failed files list output to parallel pipeline --- batbot/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/batbot/__init__.py b/batbot/__init__.py index bbe8219..385b3cf 100644 --- a/batbot/__init__.py +++ b/batbot/__init__.py @@ -67,6 +67,7 @@ def pipeline( fast_mode=False, force_overwrite=False, quiet=False, + debug=False, ): """ Run the ML pipeline on a given WAV filepath and return the classification results @@ -105,6 +106,7 @@ def pipeline( fast_mode=fast_mode, force_overwrite=force_overwrite, quiet=quiet, + debug=debug ) return output_paths, compressed_paths, metadata_path @@ -135,7 +137,7 @@ def pipeline_multi_wrapper( else: out_file_stems = [None]*len(filepaths) - outputs = {'output_paths': [], 'compressed_paths': [], 'metadata_paths': []} + outputs = {'output_paths': [], 'compressed_paths': [], 'metadata_paths': [], 'failed_files': []} # print(filepaths, out_file_stems) if tqdm_lock is not None: tqdm.set_lock(tqdm_lock) @@ -155,7 +157,7 @@ def pipeline_multi_wrapper( outputs['compressed_paths'].extend(compressed_paths) outputs['metadata_paths'].append(metadata_path) except: - print('WARNING: Pipeline failed on input file {}'.format(in_file)) + outputs['failed_files'].append(str(in_file)) # raise return tuple(outputs.values()) @@ -186,7 +188,7 @@ def parallel_pipeline( num_workers = min(len(in_file_chunks), num_workers) - outputs = {'output_paths': [], 'compressed_paths': [], 'metadata_paths': []} + outputs = {'output_paths': [], 'compressed_paths': [], 'metadata_paths': [], 'failed_files': []} lock_manager = Manager() tqdm_lock = lock_manager.Lock() @@ -205,10 +207,11 @@ def parallel_pipeline( for index, (file_chunk, out_stem_chunk) in enumerate(zip(in_file_chunks, out_stem_chunks))] for future in concurrent.futures.as_completed(futures): - output_paths, compressed_paths, metadata_path = future.result() + output_paths, compressed_paths, metadata_path, failed_files = future.result() outputs['output_paths'].extend(output_paths) outputs['compressed_paths'].extend(compressed_paths) outputs['metadata_paths'].extend(metadata_path) + outputs['failed_files'].extend(failed_files) progress.update(1) return tuple(outputs.values()) From 9b832baec9a6cf6a5526cfbb6dd07f1271ebd15f Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 18:32:02 -0500 Subject: [PATCH 20/33] Handling zero-valued STFT when converting to dB --- batbot/spectrogram/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index 862fc7d..f17ac09 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -276,7 +276,8 @@ def load_stft( # Convert the complex power (amplitude + phase) into amplitude (decibels) # Do not threshold the data - threshold will be applied later # stft_db = librosa.power_to_db(np.abs(stft) ** 2, ref=np.max, top_db=np.inf) # OLD method, cuts off lower values - stft_db = 10 * np.log10(np.square(np.abs(stft))) + abs_sq_stft = np.square(np.abs(stft)) + stft_db = 10 * np.log10(abs_sq_stft / abs_sq_stft.max() + 1e-20) # Retrieve time vector in seconds corresponding to STFT time_vec = librosa.frames_to_time( range(stft_db.shape[1]), sr=sr, hop_length=hop_length, n_fft=n_fft From 2cf459ac0b9b4505b7fd0bfae870952e898d7993 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 18:37:06 -0500 Subject: [PATCH 21/33] Formatting click help --- batbot/batbot_cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/batbot/batbot_cli.py b/batbot/batbot_cli.py index 659821b..0606887 100755 --- a/batbot/batbot_cli.py +++ b/batbot/batbot_cli.py @@ -138,7 +138,7 @@ def pipeline( ) @click.option( '--output-json', - help='Path to output JSON (if unspecified, output file locations are printed to screen)', + help='Path to output JSON (if unspecified, output file locations are printed to screen).', default=None, type=str, ) @@ -163,6 +163,8 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor will recursively search through the directory and all subfolders to find all contained *.wav files. Alternatively, the argument can be given as a string using wildcard ** for folders and/or * in filenames (if ** wildcard is used, will recursively search through all subfolders). + + \b Examples: batbot preprocess ../data -o ./tmp batbot preprocess "../data/**/*.wav" From 046f9fb5f1ce59cf0424aafa25555534be5a965c Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 18:52:53 -0500 Subject: [PATCH 22/33] Linting, and added printing exceptions for failed files --- batbot/batbot_cli.py | 116 ++++++++++++++++++++++++++++++++----------- 1 file changed, 86 insertions(+), 30 deletions(-) diff --git a/batbot/batbot_cli.py b/batbot/batbot_cli.py index 0606887..f038b81 100755 --- a/batbot/batbot_cli.py +++ b/batbot/batbot_cli.py @@ -2,23 +2,32 @@ """ CLI for BatBot """ -from glob import glob import json -from os.path import exists, commonpath, join, relpath, split, splitext, basename, isdir, isfile -from os import makedirs, getcwd, remove import warnings +from glob import glob +from os import getcwd, makedirs, remove +from os.path import ( + basename, + commonpath, + exists, + isdir, + isfile, + join, + relpath, + split, + splitext, +) import click import numpy as np +from tqdm import tqdm import batbot from batbot import log -from tqdm import tqdm - - # warnings.filterwarnings("error") + def pipeline_filepath_validator(ctx, param, value): if not exists(value): log.error(f'Input filepath does not exist: {value}') @@ -106,6 +115,7 @@ def pipeline( else: print(data) + @click.command('preprocess') @click.argument( 'filepaths', @@ -113,24 +123,28 @@ def pipeline( type=str, ) @click.option( - '--output-dir', '-o', + '--output-dir', + '-o', help='Processed file root output directory. Outputs will attempt to mirror input file directory structure if given multiple inputs (unless --no-file-structure flag is given). Defaults to current working directory.', nargs=1, default='.', type=str, ) @click.option( - '--process-metadata', '-m', + '--process-metadata', + '-m', help='Use a slower version of the pipeline which increases spectogram compression quality and also outputs bat call metadata.', is_flag=True, ) @click.option( - '--force-overwrite', '-f', + '--force-overwrite', + '-f', help='Force overwriting of compressed spectrogram and other output files.', is_flag=True, ) @click.option( - '--num-workers', '-n', + '--num-workers', + '-n', help='Number of parallel workers to use. Set to zero for serial computation only.', nargs=1, default=0, @@ -143,7 +157,8 @@ def pipeline( type=str, ) @click.option( - '--dry-run', '-d', + '--dry-run', + '-d', help='List out all the audio files to be loaded and all the anticipated output files. Additionally lists all "extra" files in the output directory that would be deleted if using the --cleanup flag.', is_flag=True, ) @@ -157,13 +172,23 @@ def pipeline( help='(Not recommended) Turn off input file directory structure mirroring. All outputs will be written directly into the provided output dir. WARNING: If multiple input files have the same filename, outputs will overwrite!', is_flag=True, ) -def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_workers, output_json, dry_run, cleanup, no_file_structure): - """Generate compressed spectrogram images for wav files into the current working directory. +def preprocess( + filepaths, + output_dir, + process_metadata, + force_overwrite, + num_workers, + output_json, + dry_run, + cleanup, + no_file_structure, +): + """Generate compressed spectrogram images for wav files into the current working directory. Takes one or more space separated arguments of filepaths to process. If given a directory name, will recursively search through the directory and all subfolders to find all contained *.wav files. Alternatively, the argument can be given as a string using wildcard ** for folders and/or * in filenames (if ** wildcard is used, will recursively search through all subfolders). - + \b Examples: batbot preprocess ../data -o ./tmp @@ -176,7 +201,7 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor in_filepaths = [] for file in filepaths: if isdir(file): - in_filepaths.extend(glob(join(file,'**/*.wav'), recursive=True)) + in_filepaths.extend(glob(join(file, '**/*.wav'), recursive=True)) elif isfile(file): in_filepaths.append(file) else: @@ -195,7 +220,9 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor if no_file_structure: out_filepath_stems = [join(root_outpath, splitext(x)[0]) for x in in_filepaths] else: - out_filepath_stems = [splitext(join(root_outpath, relpath(x, root_inpath)))[0] for x in in_filepaths] + out_filepath_stems = [ + splitext(join(root_outpath, relpath(x, root_inpath)))[0] for x in in_filepaths + ] new_dirs = [split(x)[0] for x in out_filepath_stems] for new_dir in set(new_dirs): makedirs(new_dir, exist_ok=True) @@ -217,7 +244,11 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor out_filepath_stems = out_filepath_stems[np.invert(idx_remove)] n_skipped = sum(idx_remove) if len(in_filepaths) == 0: - print('Found no unprocessed files given filepaths input {} and output directory "{}" after skipping {} files'.format(filepaths, root_outpath, n_skipped)) + print( + 'Found no unprocessed files given filepaths input {} and output directory "{}" after skipping {} files'.format( + filepaths, root_outpath, n_skipped + ) + ) print('If desired, use --force-overwrite flag to overwrite existing processed data') return @@ -247,18 +278,29 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor print('\tFlattening output file structure') print('\tCurrent working dir: {}'.format(getcwd())) print('\tOutput root dir: {}'.format(output_dir)) - print('\tFirst input file -> output files: {} -> {}.*'.format(in_filepaths[0], out_filepath_stems[0])) + print( + '\tFirst input file -> output files: {} -> {}.*'.format( + in_filepaths[0], out_filepath_stems[0] + ) + ) if len(in_filepaths) > 2: - print('\tLast input file -> output files: {} -> {}.*'.format(in_filepaths[-1], out_filepath_stems[-1])) + print( + '\tLast input file -> output files: {} -> {}.*'.format( + in_filepaths[-1], out_filepath_stems[-1] + ) + ) if dry_run: # Print out files to be processed, anticipated outputs, and files that would be deleted in cleanup mode. print('\nDry run mode active - skipping all processing') data = {} - data['input file, output file stem'] = [(str(x),'{}.*'.format(y)) for x, y in zip(in_filepaths, out_filepath_stems)] + data['input file, output file stem'] = [ + (str(x), '{}.*'.format(y)) for x, y in zip(in_filepaths, out_filepath_stems) + ] data['files to be deleted in cleanup'] = list(extra_files) if output_json is None: import pprint + pprint.pp(data) else: with open(output_json, 'w') as outfile: @@ -266,13 +308,17 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor print('Outputs written to {}'.format(output_json)) print('Complete.') return - + if cleanup: print('\nCleanup mode active - skipping all processing') if len(extra_files) == 0: print('No files to delete') else: - usr_in = input('Found {} files to delete (recommend to see details by running with --dry-run flag). Continue (y/n)? '.format(len(extra_files))) + usr_in = input( + 'Found {} files to delete (recommend to see details by running with --dry-run flag). Continue (y/n)? '.format( + len(extra_files) + ) + ) if usr_in.lower() not in ['y', 'yes']: print('Aborting cleanup mode.') return @@ -281,16 +327,20 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor remove(file) print('Complete.') return - + # Begin execution loop. - data = {'output_path':[], 'compressed_path':[], 'metadata_path':[], 'failed_files':[]} + data = {'output_path': [], 'compressed_path': [], 'metadata_path': [], 'failed_files': []} if num_workers is None or num_workers == 0: # Serial execution. - for file, out_stem in tqdm(zip(in_filepaths, out_filepath_stems), desc='Preprocessing files', total=len(in_filepaths)): + for file, out_stem in tqdm( + zip(in_filepaths, out_filepath_stems), + desc='Preprocessing files', + total=len(in_filepaths), + ): try: output_paths, compressed_paths, metadata_path = batbot.pipeline( - file, + file, out_file_stem=out_stem, fast_mode=(not process_metadata), force_overwrite=force_overwrite, @@ -300,9 +350,9 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor data['compressed_path'].extend(compressed_paths) if process_metadata: data['metadata_path'].append(metadata_path) - except: + except Exception as e: warnings.warn('WARNING: Pipeline failed for file {}'.format(file)) - data['failed_files'].append(str(file)) + data['failed_files'].append((str(file), e)) # raise else: # Parallel execution. @@ -310,7 +360,12 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor zipped = np.stack((in_filepaths, out_filepath_stems), axis=-1) np.random.seed(0) np.random.shuffle(zipped) - assert all([x in zipped[:,0] and y in zipped[:,1] for x, y in zip(in_filepaths, out_filepath_stems)]) + assert all( + [ + x in zipped[:, 0] and y in zipped[:, 1] + for x, y in zip(in_filepaths, out_filepath_stems) + ] + ) in_filepaths, out_filepath_stems = zipped.T assert all([basename(y) in basename(x) for x, y in zip(in_filepaths, out_filepath_stems)]) @@ -337,6 +392,7 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor if output_json is None: import pprint + print('\nFull spectrogram output paths:') pprint.pp(sorted(data['output_path'])) print('\nCompressed spectrogram output paths:') @@ -344,7 +400,7 @@ def preprocess(filepaths, output_dir, process_metadata, force_overwrite, num_wor if process_metadata: print('\nProcessed metadata paths:') pprint.pp(sorted(data['metadata_path'])) - print('\nFiles that failed processing and were skipped:') + print('\nFiles skipped due to failure, and corresponding exceptions:') pprint.pp(sorted(data['failed_files'])) else: with open(output_json, 'w') as outfile: From c1fd0170a09b0631d9b5f85c91969dc5d8a2e7e3 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 18:54:01 -0500 Subject: [PATCH 23/33] Linting --- batbot/__init__.py | 81 +++++++++++++++++++------------- batbot/spectrogram/__init__.py | 84 ++++++++++++++++++++++++---------- 2 files changed, 108 insertions(+), 57 deletions(-) diff --git a/batbot/__init__.py b/batbot/__init__.py index 385b3cf..12742e3 100644 --- a/batbot/__init__.py +++ b/batbot/__init__.py @@ -18,8 +18,8 @@ """ import concurrent.futures -from os.path import exists, join, split, splitext from multiprocessing import Manager +from os.path import exists, join, split, splitext from pathlib import Path import pooch @@ -101,16 +101,18 @@ def pipeline( # default to writing directly to working directory out_file_stem = splitext(split(filepath)[-1])[0] # Generate spectrogram - output_paths, compressed_paths, metadata_path, metadata = spectrogram.compute(filepath, - out_file_stem=out_file_stem, - fast_mode=fast_mode, - force_overwrite=force_overwrite, - quiet=quiet, - debug=debug - ) + output_paths, compressed_paths, metadata_path, metadata = spectrogram.compute( + filepath, + out_file_stem=out_file_stem, + fast_mode=fast_mode, + force_overwrite=force_overwrite, + quiet=quiet, + debug=debug, + ) return output_paths, compressed_paths, metadata_path + def pipeline_multi_wrapper( filepaths, out_file_stems=None, @@ -118,7 +120,7 @@ def pipeline_multi_wrapper( force_overwrite=False, worker_position=None, quiet=False, - tqdm_lock=None + tqdm_lock=None, ): """Fault-tolerant wrapper for multiple inputs. @@ -133,35 +135,41 @@ def pipeline_multi_wrapper( """ if out_file_stems is not None: - assert len(filepaths) == len(out_file_stems), 'Input filepaths and out_file_stems have different length.' + assert len(filepaths) == len( + out_file_stems + ), 'Input filepaths and out_file_stems have different length.' else: - out_file_stems = [None]*len(filepaths) + out_file_stems = [None] * len(filepaths) outputs = {'output_paths': [], 'compressed_paths': [], 'metadata_paths': [], 'failed_files': []} # print(filepaths, out_file_stems) if tqdm_lock is not None: tqdm.set_lock(tqdm_lock) - for in_file, out_stem in tqdm(zip(filepaths, out_file_stems), - desc='Processing, worker {}'.format(worker_position), - position=worker_position, - total=len(filepaths), - leave=True): + for in_file, out_stem in tqdm( + zip(filepaths, out_file_stems), + desc='Processing, worker {}'.format(worker_position), + position=worker_position, + total=len(filepaths), + leave=True, + ): try: output_paths, compressed_paths, metadata_path = pipeline( in_file, out_file_stem=out_stem, fast_mode=fast_mode, force_overwrite=force_overwrite, - quiet=quiet) + quiet=quiet, + ) outputs['output_paths'].extend(output_paths) outputs['compressed_paths'].extend(compressed_paths) outputs['metadata_paths'].append(metadata_path) - except: - outputs['failed_files'].append(str(in_file)) + except Exception as e: + outputs['failed_files'].append((str(in_file), e)) # raise return tuple(outputs.values()) + def parallel_pipeline( in_file_chunks, out_stem_chunks=None, @@ -174,12 +182,14 @@ def parallel_pipeline( ): if out_stem_chunks is None: - out_stem_chunks = [None]*len(in_file_chunks) + out_stem_chunks = [None] * len(in_file_chunks) if len(in_file_chunks) == 0: return None else: - assert len(in_file_chunks) == len(out_stem_chunks), 'in_file_chunks and out_stem_chunks must have the same length.' + assert len(in_file_chunks) == len( + out_stem_chunks + ), 'in_file_chunks and out_stem_chunks must have the same length.' if threaded: executor_cls = concurrent.futures.ThreadPoolExecutor @@ -196,15 +206,21 @@ def parallel_pipeline( with tqdm(total=len(in_file_chunks), disable=quiet, desc=desc) as progress: with executor_cls(max_workers=num_workers) as executor: - futures = [executor.submit(pipeline_multi_wrapper, - filepaths=file_chunk, - out_file_stems=out_stem_chunk, - fast_mode=fast_mode, - force_overwrite=force_overwrite, - worker_position=index % num_workers, - quiet=quiet, - tqdm_lock=tqdm_lock) - for index, (file_chunk, out_stem_chunk) in enumerate(zip(in_file_chunks, out_stem_chunks))] + futures = [ + executor.submit( + pipeline_multi_wrapper, + filepaths=file_chunk, + out_file_stems=out_stem_chunk, + fast_mode=fast_mode, + force_overwrite=force_overwrite, + worker_position=index % num_workers, + quiet=quiet, + tqdm_lock=tqdm_lock, + ) + for index, (file_chunk, out_stem_chunk) in enumerate( + zip(in_file_chunks, out_stem_chunks) + ) + ] for future in concurrent.futures.as_completed(futures): output_paths, compressed_paths, metadata_path, failed_files = future.result() @@ -213,9 +229,10 @@ def parallel_pipeline( outputs['metadata_paths'].extend(metadata_path) outputs['failed_files'].extend(failed_files) progress.update(1) - + return tuple(outputs.values()) + def batch( filepaths, config=None, @@ -270,7 +287,7 @@ def example(): wav_filepath = join(PWD, 'examples', 'example1.wav') # wav_filepath = join(PWD, 'examples', 'extras', '3517_NE_20220622_220344_814.wav') - # wav_filepath = join(PWD, 'examples', 'extras', 'p33_g67260_f29842117.wav') + # wav_filepath = join(PWD, 'examples', 'extras', 'p33_g67260_f29842117.wav') if not exists(wav_filepath): wav_filepath = pooch.retrieve( diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index f17ac09..03529b2 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -1,10 +1,10 @@ """ """ -from glob import glob import json import os import shutil import warnings +from glob import glob from os.path import basename, exists, join, split import cv2 @@ -14,14 +14,13 @@ # import networkx as nx import numpy as np import pyastar2d +import scipy.stats import tqdm from line_profiler import LineProfiler from scipy import ndimage # from PIL import Image from scipy.ndimage import gaussian_filter1d, median_filter -import scipy.stats - from shapely.geometry import Point from shapely.geometry.polygon import Polygon from skimage import draw, measure @@ -308,10 +307,10 @@ def load_stft( # Estimate maximum frequency band containing data based on original sample rate # Only data up to this maximum band should be used when computing statistics - max_band_idx = min((int(np.where(bands < orig_sr / 2.02)[0][-1]), len(bands)-1)) + max_band_idx = min((int(np.where(bands < orig_sr / 2.02)[0][-1]), len(bands) - 1)) # set non-physical noise above the max band to a minimum value - if max_band_idx < len(bands)-1: - stft_db[max_band_idx+1:, :] = np.min(stft_db[:max_band_idx+1, :]) + if max_band_idx < len(bands) - 1: + stft_db[max_band_idx + 1 :, :] = np.min(stft_db[: max_band_idx + 1, :]) return stft_db, waveplot, sr, bands, duration, min_index, time_vec, orig_sr, max_band_idx @@ -331,7 +330,7 @@ def gain_stft(stft_db, gain_db=120.0, autogain_stddev=5.0, max_band_idx=None): # Calculate the non-zero median DB and MAD # autogain signal if (median - alpha * deviation) is higher than provided gain if max_band_idx is not None: - temp = stft_db[:max_band_idx+1,:][stft_db[:max_band_idx+1,:] > 0] + temp = stft_db[: max_band_idx + 1, :][stft_db[: max_band_idx + 1, :] > 0] else: temp = stft_db[stft_db > 0] med_db = np.median(temp) @@ -586,7 +585,14 @@ def calculate_mean_within_stddev_window(values, window): def tighten_ranges( - stft_db, ranges, window, duration, skew_stddev=2.0, min_duration_ms=2.0, output_path='.', quiet=False + stft_db, + ranges, + window, + duration, + skew_stddev=2.0, + min_duration_ms=2.0, + output_path='.', + quiet=False, ): minimum_duration = int(np.around(stft_db.shape[1] / (duration * 1e3) * min_duration_ms)) @@ -755,7 +761,7 @@ def threshold_contour(segment, index, output_path='.'): def filter_contour(segment, index, med_db=None, std_db=None, kernel=5, output_path='.'): # segment = cv2.erode(segment, np.ones((3, 3), np.uint8), iterations=1) - segment = median_filter(segment, size=(kernel,kernel), mode='reflect') + segment = median_filter(segment, size=(kernel, kernel), mode='reflect') if None not in {med_db, std_db}: segment_threshold = med_db - std_db @@ -1230,14 +1236,14 @@ def find_contour_and_peak( threshold = peak_db - threshold_std * peak_db_std # pad all edges to handle signal that butts up against segment edges - segment_pad = np.pad(segment, ((2,2),(2,2))) + segment_pad = np.pad(segment, ((2, 2), (2, 2))) contours = measure.find_contours( segment_pad, level=threshold, fully_connected='high', positive_orientation='high' ) # remove padding in output contour for contour in contours: - contour[:,0] -= 2 - contour[:,1] -= 2 + contour[:, 0] -= 2 + contour[:, 1] -= 2 # Display the image and plot all contours found if output_path: @@ -1410,12 +1416,17 @@ def compute_wrapper( test_file = '{}.*'.format(out_file_stem) test_glob = glob(test_file) if len(test_glob) > 0: - print('NOTE: Found existing file(s) at {} with force_overwrite=False. Skipping file.'.format(test_file)) + if not quiet: + print( + 'NOTE: Found existing file(s) at {} with force_overwrite=False. Skipping file.'.format( + test_file + ) + ) return [], [], [], {} if fast_mode: bitdepth = 8 - quiet=True + quiet = True assert bitdepth in [8, 16] dtype = np.uint8 if bitdepth == 8 else np.uint16 @@ -1427,8 +1438,8 @@ def compute_wrapper( with warnings.catch_warnings(): warnings.simplefilter('ignore', category=DeprecationWarning) # ignore warning due to aifc deprecation - stft_db, waveplot, sr, bands, duration, freq_offset, time_vec, orig_sr, max_band_idx = load_stft( - wav_filepath, fast_mode=fast_mode + stft_db, waveplot, sr, bands, duration, freq_offset, time_vec, orig_sr, max_band_idx = ( + load_stft(wav_filepath, fast_mode=fast_mode) ) # Apply a dynamic range to a fixed dB range @@ -1449,7 +1460,11 @@ def compute_wrapper( if not fast_mode: # Plot the histogram, ignoring any non-zero values (will no-op if output_path is None) global_med_db, global_std_db, global_peak_db = plot_histogram( - stft_db, ignore_zeros=True, smoothing=512, min_band_idx=min_band_idx, output_path=debug_path + stft_db, + ignore_zeros=True, + smoothing=512, + min_band_idx=min_band_idx, + output_path=debug_path, ) # Estimate a global threshold for finding the edges of bat call contours global_threshold_std = 2.0 @@ -1472,7 +1487,11 @@ def compute_wrapper( time_vec=time_vec, ) candidates, candidate_max_dbs = create_coarse_candidates( - stft_db, window, stride, threshold_stddev=threshold_stddev, min_band_idx=min_band_idx, + stft_db, + window, + stride, + threshold_stddev=threshold_stddev, + min_band_idx=min_band_idx, ) if fast_mode: @@ -1486,9 +1505,15 @@ def compute_wrapper( candidate_max_dbs = [] # Filter all candidates to the ranges that have a substantial right-side skew - + ranges, reject_idxs = filter_candidates_to_ranges( - stft_db, candidates, area_percent=0.01, min_band_idx=min_band_idx, output_path=debug_path, fast_mode=fast_mode, quiet=quiet + stft_db, + candidates, + area_percent=0.01, + min_band_idx=min_band_idx, + output_path=debug_path, + fast_mode=fast_mode, + quiet=quiet, ) # Add in user-specified annotations to ranges @@ -1535,7 +1560,9 @@ def compute_wrapper( else: # Tighten the ranges by looking for substantial right-side skew (use stride for a smaller sampling window) - ranges = tighten_ranges(stft_db, ranges, stride, duration, output_path=debug_path, quiet=quiet) + ranges = tighten_ranges( + stft_db, ranges, stride, duration, output_path=debug_path, quiet=quiet + ) # Extract chirp metrics and metadata segments = { @@ -1613,7 +1640,12 @@ def compute_wrapper( min_bandwidth_khz = 1e3 min_duration_ms = 2.0 bandwidth, duration_, significant = significant_contour_path( - call_begin, call_end, y_step_freq, x_step_ms, min_bandwidth_khz=min_bandwidth_khz, min_duration_ms=min_duration_ms + call_begin, + call_end, + y_step_freq, + x_step_ms, + min_bandwidth_khz=min_bandwidth_khz, + min_duration_ms=min_duration_ms, ) if not significant: continue @@ -1678,9 +1710,11 @@ def compute_wrapper( segments['canvas'].append(canvas[:, trim_begin:trim_end]) # Update metadata with segment start and stop - start_stop = {'segment start.ms': (start + trim_begin) * x_step_ms, - 'segment end.ms': (start + trim_end) * x_step_ms, - 'segment duration.ms': (trim_end - trim_begin) * x_step_ms} + start_stop = { + 'segment start.ms': (start + trim_begin) * x_step_ms, + 'segment end.ms': (start + trim_end) * x_step_ms, + 'segment duration.ms': (trim_end - trim_begin) * x_step_ms, + } metadata.update(start_stop) # Normalize values From b791430c4664c83a0e225c1c12811bda2e6a6af8 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 19:02:58 -0500 Subject: [PATCH 24/33] Remove LineProfiler init --- batbot/spectrogram/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index 03529b2..731f3b8 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -27,7 +27,7 @@ from batbot import log -lp = LineProfiler() +# lp = LineProfiler() FREQ_MIN = 5e3 From 0d372bf0d89db246c63a449e46e8ddf541b7f91b Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 19:13:34 -0500 Subject: [PATCH 25/33] Renaming file for merge --- batbot/{batbot_cli.py => batbot.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename batbot/{batbot_cli.py => batbot.py} (100%) diff --git a/batbot/batbot_cli.py b/batbot/batbot.py similarity index 100% rename from batbot/batbot_cli.py rename to batbot/batbot.py From e2a909a81ec2d85c147270cab4a28a591265e10b Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 19:34:44 -0500 Subject: [PATCH 26/33] Renaming to allow for IDE debug --- batbot/{batbot.py => batbot_cli.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename batbot/{batbot.py => batbot_cli.py} (100%) diff --git a/batbot/batbot.py b/batbot/batbot_cli.py similarity index 100% rename from batbot/batbot.py rename to batbot/batbot_cli.py From ae77da836e3728e59fd6a4d24a85b9184be3f61e Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 19:36:20 -0500 Subject: [PATCH 27/33] Linting --- batbot/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/batbot/__init__.py b/batbot/__init__.py index 2b42641..59a188e 100644 --- a/batbot/__init__.py +++ b/batbot/__init__.py @@ -19,7 +19,7 @@ import concurrent.futures from multiprocessing import Manager -from os.path import exists, join, split, splitext, basename +from os.path import basename, exists, join, split, splitext from pathlib import Path import pooch @@ -303,7 +303,9 @@ def example(): output_stem = join('output', splitext(basename(wav_filepath))[0]) start_time = time.time() - results = pipeline(wav_filepath, out_file_stem=output_stem, fast_mode=False, force_overwrite=True) + results = pipeline( + wav_filepath, out_file_stem=output_stem, fast_mode=False, force_overwrite=True + ) stop_time = time.time() print('Example pipeline completed in {} seconds.'.format(stop_time - start_time)) From 6c3f20fee4c98a91e94aefd4af88c6f6dcd37529 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 19:36:34 -0500 Subject: [PATCH 28/33] Removing unused import --- batbot/spectrogram/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index 6789ea6..85d71e0 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -16,7 +16,7 @@ import pyastar2d import scipy.stats import tqdm -from line_profiler import LineProfiler +# from line_profiler import LineProfiler from scipy import ndimage # from PIL import Image From 7ecf1c4c719b5c47e43cf724f4b83f94cb71018f Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 19:37:22 -0500 Subject: [PATCH 29/33] Linting --- batbot/spectrogram/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index 85d71e0..a078a20 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -16,6 +16,7 @@ import pyastar2d import scipy.stats import tqdm + # from line_profiler import LineProfiler from scipy import ndimage From 23089e86e1ccb600d4fc608779006395b61c6e4a Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 19:48:26 -0500 Subject: [PATCH 30/33] Updated for new compute argument --- tests/test_spectrogram.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_spectrogram.py b/tests/test_spectrogram.py index 47fc1dc..deec324 100644 --- a/tests/test_spectrogram.py +++ b/tests/test_spectrogram.py @@ -1,4 +1,4 @@ -from os.path import abspath, join +from os.path import abspath, basename, join, splitext def test_spectrogram_compute(): @@ -6,4 +6,5 @@ def test_spectrogram_compute(): wav_filepath = abspath(join('examples', 'example2.wav')) output_folder = './output' - output_paths, metadata_path, metadata = compute(wav_filepath, output_folder=output_folder) + output_stem = join(output_folder, splitext(basename(wav_filepath))[0]) + output_paths, compressed_paths, metadata_path, metadata = compute(wav_filepath, out_file_stem=output_stem) From e701ddb0e1a33bf76c1a43fef5a1c9c42b629606 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 19:49:21 -0500 Subject: [PATCH 31/33] Linting --- tests/test_spectrogram.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_spectrogram.py b/tests/test_spectrogram.py index deec324..beea16d 100644 --- a/tests/test_spectrogram.py +++ b/tests/test_spectrogram.py @@ -7,4 +7,6 @@ def test_spectrogram_compute(): wav_filepath = abspath(join('examples', 'example2.wav')) output_folder = './output' output_stem = join(output_folder, splitext(basename(wav_filepath))[0]) - output_paths, compressed_paths, metadata_path, metadata = compute(wav_filepath, out_file_stem=output_stem) + output_paths, compressed_paths, metadata_path, metadata = compute( + wav_filepath, out_file_stem=output_stem + ) From 27cb03ec668dd15a1f887a49febc7ef3581f0987 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Wed, 28 Jan 2026 20:10:21 -0500 Subject: [PATCH 32/33] Combine support for output folder and/or filename stem --- batbot/__init__.py | 8 ++++---- batbot/batbot_cli.py | 5 +---- batbot/spectrogram/__init__.py | 15 ++++++++++++--- tests/test_spectrogram.py | 5 ++--- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/batbot/__init__.py b/batbot/__init__.py index 59a188e..d85f744 100644 --- a/batbot/__init__.py +++ b/batbot/__init__.py @@ -19,7 +19,7 @@ import concurrent.futures from multiprocessing import Manager -from os.path import basename, exists, join, split, splitext +from os.path import basename, exists, join, splitext from pathlib import Path import pooch @@ -64,6 +64,7 @@ def fetch(pull=False, config=None): def pipeline( filepath, out_file_stem=None, + output_folder=None, fast_mode=False, force_overwrite=False, quiet=False, @@ -97,13 +98,12 @@ def pipeline( Returns: tuple ( float, list ( dict ) ): classifier score, list of time windows """ - if out_file_stem is None: - # default to writing directly to working directory - out_file_stem = splitext(split(filepath)[-1])[0] + # Generate spectrogram output_paths, compressed_paths, metadata_path, metadata = spectrogram.compute( filepath, out_file_stem=out_file_stem, + output_folder=output_folder, fast_mode=fast_mode, force_overwrite=force_overwrite, quiet=quiet, diff --git a/batbot/batbot_cli.py b/batbot/batbot_cli.py index 17e35f2..a9db585 100755 --- a/batbot/batbot_cli.py +++ b/batbot/batbot_cli.py @@ -82,14 +82,11 @@ def pipeline( # classifier_thresh, ): - # define out file stem using given output folder - out_file_stem = join(output_path, splitext(basename(filepath))[0]) - batbot.pipeline( filepath, # config=config, # classifier_thresh=classifier_thresh, - out_file_stem=out_file_stem, + output_folder=output_path, ) diff --git a/batbot/spectrogram/__init__.py b/batbot/spectrogram/__init__.py index a078a20..99fb956 100644 --- a/batbot/spectrogram/__init__.py +++ b/batbot/spectrogram/__init__.py @@ -5,7 +5,7 @@ import shutil import warnings from glob import glob -from os.path import basename, exists, join, split +from os.path import basename, exists, join, split, splitext import cv2 import librosa @@ -1384,6 +1384,7 @@ def calculate_harmonic_and_echo_flags( def compute_wrapper( wav_filepath, out_file_stem=None, + output_folder=None, fast_mode=False, force_overwrite=False, quiet=False, @@ -1433,9 +1434,17 @@ def compute_wrapper( chunksize = int(50e3) - output_folder = split(out_file_stem)[0] + # Default to retrieving the output_folder from out_file_stem + if out_file_stem is not None: + output_folder = split(out_file_stem)[0] + if output_folder is None: + output_folder = './output' + # If no out_file_stem is given, default to the wav file basename joined with output_folder + if out_file_stem is None: + out_file_stem = join(output_folder, splitext(basename(wav_filepath))[0]) + debug_path = get_debug_path(output_folder, wav_filepath, enabled=debug) - # create output folder if it doesn't exist + # Create output folder if it doesn't exist if not os.path.exists(output_folder): os.makedirs(output_folder) assert exists(output_folder) diff --git a/tests/test_spectrogram.py b/tests/test_spectrogram.py index beea16d..5120082 100644 --- a/tests/test_spectrogram.py +++ b/tests/test_spectrogram.py @@ -1,4 +1,4 @@ -from os.path import abspath, basename, join, splitext +from os.path import abspath, join def test_spectrogram_compute(): @@ -6,7 +6,6 @@ def test_spectrogram_compute(): wav_filepath = abspath(join('examples', 'example2.wav')) output_folder = './output' - output_stem = join(output_folder, splitext(basename(wav_filepath))[0]) output_paths, compressed_paths, metadata_path, metadata = compute( - wav_filepath, out_file_stem=output_stem + wav_filepath, output_folder=output_folder ) From 1eeab630c7fb24d8e8e4d80c603bac2e2a4a50d8 Mon Sep 17 00:00:00 2001 From: "trevor.stout" Date: Thu, 29 Jan 2026 14:14:23 -0500 Subject: [PATCH 33/33] Cleanup prior to merge --- batbot/__init__.py | 3 --- batbot/batbot_cli.py | 8 +------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/batbot/__init__.py b/batbot/__init__.py index d85f744..947e939 100644 --- a/batbot/__init__.py +++ b/batbot/__init__.py @@ -165,7 +165,6 @@ def pipeline_multi_wrapper( outputs['metadata_paths'].append(metadata_path) except Exception as e: outputs['failed_files'].append((str(in_file), e)) - # raise return tuple(outputs.values()) @@ -286,8 +285,6 @@ def example(): TEST_WAV_HASH = '391efce5433d1057caddb4ce07b9712c523d6a815e4ee9e64b62973569982925' # NOQA wav_filepath = join(PWD, 'examples', 'example1.wav') - # wav_filepath = join(PWD, 'examples', 'extras', '3517_NE_20220622_220344_814.wav') - # wav_filepath = join(PWD, 'examples', 'extras', 'p33_g67260_f29842117.wav') if not exists(wav_filepath): wav_filepath = pooch.retrieve( diff --git a/batbot/batbot_cli.py b/batbot/batbot_cli.py index a9db585..eb24588 100755 --- a/batbot/batbot_cli.py +++ b/batbot/batbot_cli.py @@ -3,6 +3,7 @@ CLI for BatBot """ import json +import pprint import warnings from glob import glob from os import getcwd, makedirs, remove @@ -25,8 +26,6 @@ import batbot from batbot import log -# warnings.filterwarnings("error") - def pipeline_filepath_validator(ctx, param, value): if not exists(value): @@ -273,8 +272,6 @@ def preprocess( ] data['files to be deleted in cleanup'] = list(extra_files) if output_json is None: - import pprint - pprint.pp(data) else: with open(output_json, 'w') as outfile: @@ -327,7 +324,6 @@ def preprocess( except Exception as e: warnings.warn('WARNING: Pipeline failed for file {}'.format(file)) data['failed_files'].append((str(file), e)) - # raise else: # Parallel execution. # shuffle input and output paths @@ -365,8 +361,6 @@ def preprocess( data['failed_files'].extend(failed_files) if output_json is None: - import pprint - print('\nFull spectrogram output paths:') pprint.pp(sorted(data['output_path'])) print('\nCompressed spectrogram output paths:')