8000
Skip to content

decoding.sh

Latest commit

 

History

History
executable file
·
1640 lines (1471 loc) · 107 KB

decoding.sh

File metadata and controls

executable file
·
1640 lines (1471 loc) · 107 KB
#!/bin/bash
############## Decoding ################################################
### For each real receiver/band there is one decode daemon and one recording daemon
### Waits for a new wav file then decodes and posts it to all of the posting client
declare -r DECODING_CLIENTS_SUBDIR="decoding_clients.d" ### Each decoding daemon will create its own subdir where it will copy YYMMDD_HHMM_wspr_spots.txt
declare MAX_ALL_WSPR_SIZE=200000 ### Delete the ALL_WSPR.TXT file once it reaches this size.. Stops wsprdaemon from filling ${WSPRDAEMON_TMP_DIR}/..
declare FFT_WINDOW_CMD=${WSPRDAEMON_ROOT_DIR}/wav_window.py
declare C2_FFT_ENABLED="yes" ### If "yes", then use the c2 file produced by wsprd to calculate FFT noise levels
declare C2_FFT_CMD=${WSPRDAEMON_ROOT_DIR}/c2_noise.py
function get_decode_mode_list() {
local modes_variable_to_return=$1
local receiver_modes_arg=$2
local receiver_band=$3
local temp_receiver_modes
temp_receiver_modes=${receiver_modes_arg}
if [[ ${receiver_modes_arg} == "DEFAULT" ]]; then
### Translate DEFAULT mode to a list of modes for this band
local default_modes=""
get_default_modes_for_band default_modes ${receiver_band}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: 'get_default_modes_for_band default_modes ${receiver_band}' => ${ret_code}"
sleep 1
return ${ret_code}
fi
wd_logger 1 "Translated decode mode '${receiver_modes_arg}' to '${default_modes}'"
temp_receiver_modes=${default_modes}
fi
### Validate the mode list
is_valid_mode_list ${temp_receiver_modes}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]] ; then
wd_logger 1 "ERROR: 'is_valid_mode_list ${temp_receiver_modes}' => ${ret_code}"
return 1
fi
wd_logger 2 "Returning modes ${temp_receiver_modes}"
eval ${modes_variable_to_return}=${temp_receiver_modes}
return 0
}
##########
function get_af_db() {
local return_variable_name=$1
local local real_receiver_name=$2 ### 'real' as opposed to 'merged' receiver
local real_receiver_rx_band=$3
local default_value
local af_info_field="$(get_receiver_af_list_from_name ${real_receiver_name})"
if [[ -z "${af_info_field}" ]]; then
wd_logger 2 "Found no AF field for receiver ${real_receiver_name}, so return AF=0"
eval ${return_variable_name}=0
return 0
fi
local af_info_list=(${af_info_field//,/ })
wd_logger 1 "af_info_list= ${af_info_list[*]}"
for element in ${af_info_list[@]}; do
local fields=(${element//:/ })
if [[ ${fields[0]} == "DEFAULT" ]]; then
default_value=${fields[1]}
wd_logger 1 "Found default value ${default_value}"
elif [[ ${fields[0]} == ${real_receiver_rx_band} ]]; then
wd_logger 1 "Found AF value ${fields[1]} for receiver ${real_receiver_name}, band ${real_receiver_rx_band}"
eval ${return_variable_name}=${fields[1]}
return 0
fi
done
wd_logger 1 "Returning default value ${default_value} for receiver ${real_receiver_name}, band ${real_receiver_rx_band}"
eval ${return_variable_name}=${default_value}
return 0
}
function calculate_nl_adjustments() {
local return_rms_corrections_variable_name=$1
local return_fft_corrections_variable_name=$2
local receiver_band=$3
local wspr_band_freq_khz=$(get_wspr_band_freq ${receiver_band})
local wspr_band_freq_mhz=$( printf "%2.4f\n" $(bc <<< "scale = 5; ${wspr_band_freq_khz}/1000.0" ) )
local wspr_band_freq_hz=$( bc <<< "scale = 0; ${wspr_band_freq_khz}*1000.0/1" )
if [[ -f ${WSPRDAEMON_ROOT_DIR}/noise_plot/noise_ca_vals.csv ]]; then
local cal_vals=($(sed -n '/^[0-9]/s/,/ /gp' ${WSPRDAEMON_ROOT_DIR}/noise_plot/noise_ca_vals.csv))
fi
### In each of these assignments, if cal_vals[] was not defined above from the file 'noise_ca_vals.csv', then use the default value. e.g. cal_c2_correction will get the default value '-187.7
local cal_nom_bw=${cal_vals[0]-320} ### In this code I assume this is 320 hertz
local cal_ne_bw=${cal_vals[1]-246}
local cal_rms_offset=${cal_vals[2]--50.4}
local cal_fft_offset=${cal_vals[3]--41.0}
local cal_fft_band=${cal_vals[4]--13.9}
local cal_threshold=${cal_vals[5]-13.1}
local cal_c2_correction=${cal_vals[6]--187.7}
local kiwi_amplitude_versus_frequency_correction="$(bc <<< "scale = 10; -1 * ( (2.2474 * (10 ^ -7) * (${wspr_band_freq_mhz} ^ 6)) - (2.1079 * (10 ^ -5) * (${wspr_band_freq_mhz} ^ 5)) + \
(7.1058 * (10 ^ -4) * (${wspr_band_freq_mhz} ^ 4)) - (1.1324 * (10 ^ -2) * (${wspr_band_freq_mhz} ^ 3)) + \
(1.0013 * (10 ^ -1) * (${wspr_band_freq_mhz} ^ 2)) - (3.7796 * (10 ^ -1) * ${wspr_band_freq_mhz} ) - (9.1509 * (10 ^ -1)))" )"
if [[ $(bc <<< "${wspr_band_freq_mhz} > 30") -eq 1 ]]; then
### Don't adjust Kiwi's af when fed by transverter
kiwi_amplitude_versus_frequency_correction=0
fi
local antenna_factor_adjust
get_af_db antenna_factor_adjust ${receiver_name} ${receiver_band}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: can't find AF for ${receiver_name} ${receiver_band}"
exit 1
fi
wd_logger 1 "Got AF = ${antenna_factor_adjust} for ${receiver_name} ${receiver_band}"
local rx_khz_offset=$(get_receiver_khz_offset_list_from_name ${receiver_name})
local total_correction_db=$(bc <<< "scale = 10; ${kiwi_amplitude_versus_frequency_correction} + ${antenna_factor_adjust}")
local calculated_rms_nl_adjust=$(bc -l <<< "var=(${cal_rms_offset} + (10 * (l( 1 / ${cal_ne_bw}) / l(10) ) ) + ${total_correction_db}); scale=2; var/1.0" ) ## bc -l invokes the math extension, l(x)/l(10) == log10(x)
wd_logger 1 "calculated_rms_nl_adjust=\$(bc -l <<< \"var=(${cal_rms_offset} + (10 * (l( 1 / ${cal_ne_bw}) / l(10) ) ) + ${total_correction_db}); scale=2; var/1.0\" )"
eval ${return_rms_corrections_variable_name}=${calculated_rms_nl_adjust}
## G3ZIL implementation of algorithm using the c2 file by Christoph Mayer
local calculated_fft_nl_adjust=$(bc <<< "scale = 2;var=${cal_c2_correction};var+=${total_correction_db}; (var * 100)/100")
wd_logger 1 "calculated_fft_nl_adjust = ${calculated_fft_nl_adjust} from calculated_fft_nl_adjust=\$(bc <<< \"scale = 2;var=${cal_c2_correction};var+=${total_correction_db}; (var * 100)/100\")"
eval ${return_fft_corrections_variable_name}="'${calculated_fft_nl_adjust}'"
}
declare WAV_SAMPLES_LIST=(
"${SIGNAL_LEVEL_PRE_TX_SEC} ${SIGNAL_LEVEL_PRE_TX_LEN}"
"${SIGNAL_LEVEL_TX_SEC} ${SIGNAL_LEVEL_TX_LEN}"
"${SIGNAL_LEVEL_POST_TX_SEC} ${SIGNAL_LEVEL_POST_TX_LEN}"
)
### Record an error line to the log file if the wav file contains audio samples which exceed these levels
declare WAV_MIN_LEVEL=${WAV_MIN_LEVEL--1.0}
declare WAV_MAX_LEVEL=${WAV_MAX_LEVEL-1.0}
function get_wav_levels()
{
local __return_levels_var=$1
local wav_filename=$2
local sample_start_sec=$3
local sample_length_secs=$4
local rms_adjust=$5
if [[ ${sample_start_sec} == ${SIGNAL_LEVEL_PRE_TX_SEC} ]]; then
### This function is called three times for each wav file. We only need to check the whole wav file once to determine the min/max values
### So execute this check only the first time
### To see if the AGC might need to change from its default 60, check to see if any samples in the whole wav file closely approach the MAX or MIN sample values
### 'sox -n stats' output this information on seperate line:
### DC offset Min level Max level Pk lev dB RMS lev dB RMS Pk dB RMS Tr dB Crest factor Flat factor Pk count Bit-depth Num samples Length s Scale max Window s
### Field #: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14
### Run 'man sox' and search for 'stats' to find a description of those statistic fields
local full_wav_stats=$(sox ${wav_filename} -n stats 2>&1) ### sox -n stats prints those to stderr
local full_wav_stats_list=( $(echo "${full_wav_stats}" | awk '{printf "%s\t", $NF }') ) ### store them in an array
if [[ ${#full_wav_stats_list[@]} -ne ${EXPECTED_SOX_STATS_FIELDS_COUNT-15} ]]; then
wd_logger 1 "ERROR: Got ${#full_wav_stats_list[@]} stats from 'sox -n stats', not the expected ${EXPECTED_SOX_STATS_FIELDS_COUNT-15} fields"
else
local full_wav_min_level=${full_wav_stats_list[1]}
local full_wav_max_level=${full_wav_stats_list[2]}
local full_wav_peak_level_count=${full_wav_stats_list[9]}
local full_wav_bit_depth=${full_wav_stats_list[10]}
local full_wav_len_secs=${full_wav_stats_list[12]}
### Min and Max level are floating point numbers and their absolute values are less than or equal to 1.0000
if [[ $( echo "${full_wav_min_level} <= ${WAV_MIN_LEVEL}" | bc ) == "1" || $( echo "${full_wav_max_level} >= ${WAV_MAX_LEVEL}" | bc ) == "1" ]] ; then
wd_logger 1 "ERROR: ${full_wav_peak_level_count} full level (+/-1.0) samples detected in file ${wav_filename} of length=${full_wav_len_secs} seconds and with Bit-depth=${full_wav_bit_depth}: the min/max levels are: min=${full_wav_min_level}, max=${full_wav_max_level}"
else
wd_logger 2 "In file ${wav_filename} of length=${full_wav_len_secs} seconds and with Bit-depth=${full_wav_bit_depth}: the min/max levels are: min=${full_wav_min_level}, max=${full_wav_max_level}"
fi
### Create a status file associated with this indsividual wav file from which the decoding daemon will extract wav overload information for the spots decoded from this wav file
echo "WAV_stats: ${full_wav_min_level} ${full_wav_max_level} ${full_wav_peak_level_count}" > ${wav_filename}.stats
### Append these stats to a log file which can be searched by a yet-to-be-implemented 'wd-...' command
local wav_status_file="${WAV_STATUS_LOG_FILE-wav_status.log}"
touch ${wav_status_file} ### In case it doesn't yet exist
if grep -q "${wav_filename}" ${wav_status_file} ; then
wd_logger 1 "ERROR: unexpectly found log line for wav file ${wav_filename} in ${wav_status_file}"
else
wd_logger 1 "Appending '${wav_filename}: ${full_wav_min_level} ${full_wav_max_level} ${full_wav_peak_level_count}' to the log file '${wav_status_file}'"
echo "${wav_filename}: ${full_wav_min_level} ${full_wav_max_level} ${full_wav_peak_level_count}" >> ${wav_status_file}
truncate_file ${wav_status_file} 100000 ### Limit the size of this log file to 100 Kb
fi
fi
fi
local wav_levels_list=( $(sox ${wav_filename} -t wav - trim ${sample_start_sec} ${sample_length_secs} 2>/dev/null | sox - -n stats 2>&1 | awk '/dB/{print $(NF)}'))
if [[ ${#wav_levels_list[@]} -ne 4 ]]; then
wd_logger 1 "ERROR: found only ${#wav_levels_list[@]} dB lines, not the four expected dB lines from 'sox ${wav_filename} -t wav - trim ${sample_start_sec} ${sample_length_secs}'"
return 1
fi
wd_logger 2 "Got sox dB values: '${wav_levels_list[*]}'"
local return_line=""
for db_val in ${wav_levels_list[@]}; do
local adjusted_val=$(bc <<< "scale = 2; (${db_val} + ${rms_adjust})/1") ### '/1' forces bc to use the scale = 2 setting
return_line="${return_line} ${adjusted_val}"
done
wd_logger 2 "Returning adjusted dB values: '${return_line}'"
eval ${__return_levels_var}=\"${return_line}\"
return 0
}
declare WAV_SECOND_RANGE=${WAV_SECOND_RANGE-10} ### wav files of +/- this number of seconds are deemed OK for wsprd to decode
declare TARGET_RAW_WAV_SECONDS=60
declare MIN_VALID_RAW_WAV_SECONDS=${MIN_VALID_RAW_WAV_SECONDS-$(( ${TARGET_RAW_WAV_SECONDS} - ${WAV_SECOND_RANGE} )) }
declare MAX_VALID_RAW_WAV_SECONDS=${MAX_VALID_RAW_WAV_SECONDS-$(( ${TARGET_RAW_WAV_SECONDS} + ${WAV_SECOND_RANGE} )) }
declare TARGET_WSPR_WAV_SECONDS=120
declare MIN_VALID_WSPR_WAV_SECONDS=${MIN_VALID_WSPR_WAV_SECONDS-$(( ${TARGET_WSPR_WAV_SECONDS} - ${WAV_SECOND_RANGE} )) }
declare MAX_VALID_WSPR_WAV_SECONDS=${MAX_VALID_WSPR_WAV_SECONDS-$(( ${TARGET_WSPR_WAV_SECONDS} + ${WAV_SECOND_RANGE} )) }
function is_valid_wav_file()
{
local wav_filename=$1
local min_valid_secs=$2
local max_valid_secs=$3
if [[ ! -f ${wav_filename} ]]; then
wd_logger 1 "ERROR: no wav file ${wav_filename}"
return 1
fi
if [[ ! -s ${wav_filename} ]]; then
wd_logger 1 "ERROR: zero length wav file ${wav_filename}"
return 1
fi
local wav_stats=$(sox ${wav_filename} -n stats 2>&1 )
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' => ${ret_code}"
return 1
fi
wd_logger 2 "'sox ${wav_filename} -n stats 2>&1' =>\n${wav_stats}"
local wav_length_line_list=( $(grep '^Length' <<< "${wav_stats}") )
if [[ ${#wav_length_line_list[@]} -eq 0 ]]; then
wd_logger 1 "ERROR: can't find wav file 'Length' line in output of 'sox ${wav_filename} -n stats'"
return 1
fi
if [[ ${#wav_length_line_list[@]} -ne 3 ]]; then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' ouput 'Length' line has ${#wav_length_line_list[@]} fields in it instead of the expected 3 fields"
return 1
fi
local wav_length_secs=${wav_length_line_list[2]/.*}
if [[ -z "${wav_length_secs}" ]]; then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' reports invalid wav file length '${wav_length_line_list[2]}'"
return 1
fi
if [[ ! ${wav_length_secs} =~ ^[0-9]+$ ]]; then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' reports wav file length ${wav_length_line_list[2]} which doesn't contain an integer number"
return 1
fi
if [[ ${wav_length_secs} -lt ${min_valid_secs} || ${wav_length_secs} -gt ${max_valid_secs} ]]; then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' reports invalid wav file length of ${wav_length_secs} seconds. valid min=${min_valid_secs}, valid max=${max_valid_secs}"
return 1
fi
return 0
}
function get_rms_levels()
{
local __return_var_name=$1
local __return_string_name=$2
local wav_filename=$3
local rms_adjust=$4
if ! is_valid_wav_file ${wav_filename} ${MIN_VALID_WSPR_WAV_SECONDS} ${MAX_VALID_WSPR_WAV_SECONDS} ; then
local rc=$?
wd_logger 1 "ERROR: 'valid_wav_file ${wav_filename}' => ${rc}"
return 1
fi
local output_line=""
local sample_info
for sample_info in "${WAV_SAMPLES_LIST[@]}"; do
local sample_line_list=( ${sample_info} )
local sample_start_sec=${sample_line_list[0]}
local sample_length_secs=${sample_line_list[1]}
local sample_vals
get_wav_levels sample_vals ${wav_filename} ${sample_start_sec} ${sample_length_secs} ${rms_adjust}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: 'get_wav_levels sample_vals ${wav_filename} ${sample_start_sec} ${sample_length_secs}' => {ret_code}"
return 1
fi
output_line="${output_line} ${sample_vals}"
done
local output_line_list=( ${output_line} )
if [[ ${#output_line_list[@]} -ne 12 ]]; then
wd_logger 1 "ERROR: expected 12 fields of dB info, but got only ${#output_line_list[@]} fields from calls to get_wav_levels()"
return 1
fi
local return_rms_value
local pre_rms_value=${output_line_list[3]} # RMS level is the minimum of the Pre and Post 'RMS Tr dB'
local post_rms_value=${output_line_list[11]} # RMS level is the minimum of the Pre and Post 'RMS Tr dB'
if [[ $(bc --mathlib <<< "${pre_rms_value} < ${post_rms_value}") -eq "1" ]]; then
return_rms_value=${pre_rms_value}
wd_logger 2 "So returning rms_level ${return_rms_value} which is from pre_tx"
else
return_rms_value=${post_rms_value}
wd_logger 2 "So returning rms_level ${return_rms_value} which is from post_tx"
fi
local signal_level_line=" ${output_line} ${return_rms_value}"
eval ${__return_var_name}=${return_rms_value}
eval ${__return_string_name}=\"${signal_level_line}\"
wd_logger 2 "Returning rms_value=${return_rms_value} and signal_level_line='${signal_level_line}'"
return 0
}
function decode_wspr_wav_file() {
local wav_file_name=$1
local wspr_decode_capture_freq_hz=$2
local rx_khz_offset=$3
local stdout_file=$4
local wsprd_cmd_flags="$5" ### ${WSPRD_CMD_FLAGS}
wd_logger 2 "Decode file ${wav_file_name} for frequency ${wspr_decode_capture_freq_hz} and send stdout to ${stdout_file}. rx_khz_offset=${rx_khz_offset}, wsprd_cmd_flags='${wsprd_cmd_flags}'"
local wspr_decode_capture_freq_hzx=${wav_file_name#*_} ### Remove the year/date/time
wspr_decode_capture_freq_hzx=${wspr_decode_capture_freq_hz%_*} ### Remove the _usb.wav
local wspr_decode_capture_freq_hzx=$( bc <<< "${wspr_decode_capture_freq_hz} + (${rx_khz_offset} * 1000)" )
local wspr_decode_capture_freq_mhz=$( printf "%2.4f\n" $(bc <<< "scale = 5; ${wspr_decode_capture_freq_hz}/1000000.0" ) )
if [[ ! -s ALL_WSPR.TXT ]]; then
touch ALL_WSPR.TXT
fi
local all_wspr_size=$(${GET_FILE_SIZE_CMD} ALL_WSPR.TXT)
if [[ ${all_wspr_size} -gt ${MAX_ALL_WSPR_SIZE} ]]; then
wd_logger 1 "ALL_WSPR.TXT has grown too large, so truncating it"
tail -n 1000 ALL_WSPR.TXT > ALL_WSPR.tmp
mv ALL_WSPR.tmp ALL_WSPR.TXT
fi
local last_line=$(tail -n 1 ALL_WSPR.TXT)
timeout ${WSPRD_TIMEOUT_SECS-110} nice ${WSPRD_CMD} -c ${wsprd_cmd_flags} -f ${wspr_decode_capture_freq_mhz} ${wav_file_name} > ${stdout_file}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: Command 'timeout ${WSPRD_TIMEOUT_SECS-110} nice ${WSPRD_CMD} -c ${wsprd_cmd_flags} -f ${wspr_decode_capture_freq_mhz} ${wav_file_name} > ${stdout_file}' returned error ${ret_code}"
return ${ret_code}
fi
grep -A 10000 "${last_line}" ALL_WSPR.TXT | grep -v "${last_line}" > ALL_WSPR.TXT.new
return ${ret_code}
}
declare WSPRD_BIN_DIR=${WSPRDAEMON_ROOT_DIR}/bin
declare WSPRD_CMD=${WSPRD_BIN_DIR}/wsprd
declare JT9_CMD=${WSPRD_BIN_DIR}/jt9
declare WSPRD_CMD_FLAGS="${WSPRD_CMD_FLAGS--C 500 -o 4 -d}"
declare WSPRD_STDOUT_FILE=wsprd_stdout.txt ### wsprd stdout goes into this file, but we use wspr_spots.txt
declare MAX_ALL_WSPR_SIZE=200000 ### Truncate the ALL_WSPR.TXT file once it reaches this size.. Stops wsprdaemon from filling ${WSPRDAEMON_TMP_DIR}/..
declare RAW_FILE_FULL_SIZE=1440000 ### Approximate number of bytes in a full size one minute long raw or wav file
### We use 'soxi' to check the length of the 1 minute long wav files created by kiwirecorder.py in a field with the form HOURS:MINUTES:SECONDS.MILLISECONDS
### Because bash can only do integer comparisons, we strip the ':'s and '.' from that field
### As a result, valid wav files will bein the ranges from 6000 - (${MIN_VALID_RAW_WAV_SECONDS} * 100) to 5999
### or in the range from 10000 to (10000 + ${MIN_VALID_RAW_WAV_SECONDS})
### So this code gets the time duration of the wave file into an integer which has the form HHMMSSUU and thus can be compared by a bash expression
### Because the field rolls over from second 59 to minute 1, There can be no fields which have the values 6000 through 9999
declare WAV_FILE_MIN_HHMMSSUU=$(( ${MIN_VALID_RAW_WAV_SECONDS} * 100 )) ### by default this = 55 seconds == 5500
declare WAV_FILE_MAX_HHMMSSUU=$(( 10000 + ( ${WAV_SECOND_RANGE} * 100) )) ### by default this = 65 seconds == 10500
### If the wav recording daemon is running, we can calculate how many seconds until it starts to fill the raw file (if 0 length first file) or fills the 2nd raw file. Sleep until then
function flush_wav_files_older_than()
{
local reference_file=$1
if [[ ! -f ${reference_file} ]]; then
wd_logger 1 "ERROR: can't find expected reference file '${reference_file}"
return 1
fi
wd_logger 1 "Delete any files older than ${reference_file}"
local olders=0
local newers=0
local wav_file
for wav_file in $(find -name '*wav'); do
if [[ ${wav_file} -ot ${reference_file} ]]; then
(( ++olders ))
wd_logger 1 "Deleting older wav file '${wav_file}'"
wd_rm ${wav_file}
elif [[ ${wav_file} -nt ${reference_file} ]]; then
(( ++newers ))
wd_logger 1 "ERROR: found wav file '${wav_file}' is newer than ${reference_file}"
else
### 'find' prepends './' to the filenames it returns, so we can't compare flenames. But if two wav file timestamps in the same directory match each other, then they must be the same wav file
wd_logger 1 "Found expected reference file ${reference_file}"
fi
done
if [[ ${olders} -gt 0 || ${newers} -gt 0 ]]; then
wd_logger 1 "Deleted ${olders} older wav files and/or found ${newers} new wav files"
fi
return 0
}
function sleep_until_raw_file_is_full() {
local filename=$1
if [[ ! -f ${filename} ]]; then
wd_logger 1 "ERROR: ${filename} doesn't exist"
return 1
fi
local old_file_size=$( ${GET_FILE_SIZE_CMD} ${filename} )
local new_file_size
local start_seconds=${SECONDS}
sleep 2
while [[ -f ${filename} ]] && new_file_size=$( ${GET_FILE_SIZE_CMD} ${filename}) && [[ ${new_file_size} -gt ${old_file_size} ]]; do
wd_logger 3 "Waiting for file ${filename} to stop growing in size. old_file_size=${old_file_size}, new_file_size=${new_file_size}"
old_file_size=${new_file_size}
sleep 2
done
local loop_seconds=$(( SECONDS - start_seconds ))
if [[ ! -f ${filename} ]]; then
wd_logger 1 "ERROR: file ${filename} disappeared after ${loop_seconds} seconds"
return 1
fi
wd_logger 1 "'${filename}' stopped growing after ${loop_seconds} seconds"
local file_start_minute=${filename:11:2}
local file_start_second=${filename:13:2}
if [[ ${file_start_second} != "00" ]]; then
wd_logger 1 "'${filename} starts at second ${file_start_second}, not at the required second '00', so delete this file which should be the first file created after startup"
flush_wav_files_older_than ${filename}
wd_rm ${filename}
return 2
fi
### Previously, I had just checked the size of the wav file to validate the duration of the recording
### My guesess of the min and max valid wav file size in bytes were too narrow and useful wav files were being thrown away
local wav_file_duration_hh_mm_sec_msec=$(soxi ${filename} | awk '/Duration/{print $3}')
local wav_file_duration_integer=$(sed 's/[\.:]//g' <<< "${wav_file_duration_hh_mm_sec_msec}")
if [[ 10#${wav_file_duration_integer} -lt ${WAV_FILE_MIN_HHMMSSUU} ]]; then ### The 10#... forces bash to treat wav_file_duration_integer as a decimal, since its leading zeros would otherwise identify it at an octal number
wd_logger 1 "The wav file stabilized at invalid too short duration ${wav_file_duration_hh_mm_sec_msec} which almost always occurs at startup. Flush this file since it can't be used as part of a WSPR wav file"
flush_wav_files_older_than ${filename}
wd_rm ${filename}
return 2
fi
if [[ 10#${wav_file_duration_integer} -gt ${WAV_FILE_MAX_HHMMSSUU} ]]; then
### If the wav file has grown to longer than one minute, then it is likely there are two kiwirecorder jobs running
### We really need to know the IP address of the Kiwi recording this band, since this freq may be recorded by other other Kiwis in a Merged group
local this_dir_path_list=( ${PWD//\// } )
local kiwi_name=${this_dir_path_list[-2]}
local kiwi_ip_addr=$(get_receiver_ip_from_name ${kiwi_name})
local kiwi_freq=${filename#*_}
kiwi_freq=${kiwi_freq::3}
local ps_output=$(ps aux | grep "${KIWI_RECORD_COMMAND}.*${kiwi_freq}.*${kiwi_ip_addr/:*}" | grep -v grep)
local kiwirecorder_pids=( $(awk '{print $2}' <<< "${ps_output}" ) )
if [[ ${#kiwirecorder_pids[@]} -eq 0 ]]; then
wd_logger 1 "ERROR: wav file stabilized at invalid too long duration ${wav_file_duration_hh_mm_sec_msec}, but can't find any kiwirecorder processes which would be creating it"
else
wd_kill ${kiwirecorder_pids[@]}
local rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: 'wd_kill ${kiwirecorder_pids[*]}' => ${rc}"
fi
wd_logger 1 "ERROR: wav file stabilized at invalid too long duration ${wav_file_duration_hh_mm_sec_msec}, so there appear to be more than one instance of the KWR running. 'ps' output was:\n${ps_output}\nSo executed 'wd_kill ${kiwirecorder_pids[*]}'"
fi
flush_wav_files_older_than ${filename}
wd_rm ${filename}
return 3
fi
wd_logger 1 "File ${filename} stabilized at size ${new_file_size} after ${loop_seconds} seconds"
return 0
}
### Returns the minute and epoch of the first sample in 'filename'. Variations in CPU and OS make using the file's timestamp a poor choice for the time source.
### So use the time in the file's name
function get_file_start_time_info()
{
local __epoch_return_variable_name=$1
local __minute_return_variable_name=$2
local file_name=$3
local epoch_from_file_stat=$( ${GET_FILE_MOD_TIME_CMD} ${file_name})
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: '${GET_FILE_MOD_TIME_CMD} ${file_name}' => ${ret_code}"
return 1
fi
local minute_from_file_epoch=$( printf "%(%M)T" ${epoch_from_file_stat} )
local year_from_file_name="${file_name:0:4}"
local month_from_file_name=${file_name:4:2}
local day_from_file_name=${file_name:6:2}
local hour_from_file_name=${file_name:9:2}
local minute_from_file_name=${file_name:11:2}
local file_spec_for_date_cmd="${month_from_file_name}/${day_from_file_name}/${year_from_file_name} ${hour_from_file_name}:${minute_from_file_name}:00"
local epoch_from_file_name=$( date --date="${file_spec_for_date_cmd}" +%s )
if [[ ${minute_from_file_epoch} != ${minute_from_file_name} ]]; then
wd_logger 1 "INFO: minute_from_file_epoch=${minute_from_file_epoch} != minute_from_file_name=${minute_from_file_name}, but always use file_name times"
fi
wd_logger 1 "File '${file_name}' => epoch_from_file_stat=${epoch_from_file_stat}, epoch_from_file_name=${epoch_from_file_name}, minute_from_file_epoch=${minute_from_file_epoch}, minute_from_file_name=${minute_from_file_name}"
eval ${__epoch_return_variable_name}=${epoch_from_file_name}
eval ${__minute_return_variable_name}=${minute_from_file_name}
return 0
}
function cleanup_wav_file_list()
{
local __return_clean_files_string_name=$1
local check_file_list=( $2 )
if [[ ${#check_file_list[@]} -eq 0 ]]; then
wd_logger 1 "Was given an empty file list"
eval ${__return_clean_files_string_name}=\"\"
return 0
fi
wd_logger 2 "Testing list of raw files: '${check_file_list[*]}'"
local last_file_minute=-1
local flush_files="no"
local test_file_name
local return_clean_files_string=""
local raw_file_index=$(( ${#check_file_list[@]} - 1 ))
while [[ ${raw_file_index} -ge 0 ]]; do
local test_file_name=${check_file_list[${raw_file_index}]}
wd_logger 2 "Testing file ${test_file_name}"
if [[ ${flush_files} == "yes" ]]; then
wd_logger 1 "ERROR: flushing file ${test_file_name}"
wd_rm ${test_file_name}
else
is_valid_wav_file ${test_file_name} ${MIN_VALID_RAW_WAV_SECONDS} ${MAX_VALID_RAW_WAV_SECONDS}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: found wav file '${test_file_name}' has invalid size. Flush it"
rm ${test_file_name}
flush_files="yes"
else
### Size is valid, see if it is one minute earlier than the previous file
local test_file_minute=${test_file_name:11:2}
wd_logger 2 "Checking time fields in valid wav file '${test_file_name}' => test_file_minute=${test_file_minute}, last_file_minute=${last_file_minute}"
if [[ ${last_file_minute} == "-1" ]]; then
wd_logger 2 "First clean file is at minute ${test_file_minute}"
last_file_minute=${test_file_minute}
return_clean_files_string="${test_file_name}"
else
if [[ 10#${last_file_minute} -eq 0 ]]; then
last_file_minute=60
wd_logger 1 "Testing for a minute 59 file '${test_file_name}', so changed last_file_minute to ${last_file_minute}"
fi
local minute_difference=$(( 10#${last_file_minute} - 10#${test_file_minute} ))
if [[ ${minute_difference} -eq 1 ]]; then
wd_logger 2 "'${test_file_name}' size is OK and it is one minute earlier than the next file in the list"
return_clean_files_string="${test_file_name} ${return_clean_files_string}"
last_file_minute=${test_file_minute}
else
wd_logger 1 "ERROR: there is a gap of more than 1 minute between this file '${test_file_name}' and the next file in the list ${check_file_list[ $(( ++${raw_file_index} )) ]}, so flush this file and all earlier files"
wd_rm ${test_file_name}
flush_files="yes"
fi
fi
fi
fi
wd_logger 2 "Done checking '${test_file_name}' from index ${raw_file_index}"
(( --raw_file_index ))
done
local clean_files_list=( ${return_clean_files_string} )
wd_logger 1 "Given check_file_list[${#check_file_list[@]}]='${check_file_list[*]}'\nReturning clean_file_list[${#clean_files_list[*]}]='${clean_files_list[*]}'"
if [[ ${#check_file_list[@]} -ne ${#clean_files_list[*]} ]]; then
wd_logger 1 "ERROR: cleaned list check_file_list[${#check_file_list[@]}]='${check_file_list[*]}' => clean_file_list[${#clean_files_list[*]}]='${clean_files_list[*]}'"
fi
eval ${__return_clea 992E n_files_string_name}=\"${return_clean_files_string}\"
return 0
}
### Waits for wav files needed to decode one or more of the MODEs have been fully recorded
function get_wav_file_list() {
local return_variable_name=$1 ### returns a string with a space-separated list each element of which is of the form MODE:first.wav[,second.wav,...]
local receiver_name=$2 ### Used when we need to start or restart the wav recording daemon
local receiver_band=$3
local receiver_modes=$4
local target_modes_list=( ${receiver_modes//:/ } ) ### Argument has form MODE1[:MODE2...] put it in local array
local -ia target_minutes_list=( $( IFS=$'\n' ; echo "${target_modes_list[*]/?/}" | sort -nu ) ) ### Chop the "W" or "F" from each mode element to get the minutes for each mode NOTE THE "s which are requried if arithmatic is being done on each element!!!!
if [[ " ${target_minutes_list[*]} " =~ " 0 " ]] ; then
### The configuration validtor verified that jobs which have mode 'W0' specified will have no other modes
### In mode W0 we are only goign to run the wsprd decoder in order to get the RMS can C2 noise levels
wd_logger 1 "Found that mode 'W0' has been specified"
target_minutes_list=( 2 )
fi
local -ia target_seconds_list=( "${target_minutes_list[@]/%/*60}" ) ### Multiply the minutes of each mode by 60 to get the number of seconds of wav files needed to decode that mode NOTE that both ' and " are needed for this to work
local oldest_file_needed=${target_seconds_list[-1]}
wd_logger 1 "Start with args '${return_variable_name} ${receiver_name} ${receiver_band} ${receiver_modes}', then receiver_modes => ${target_modes_list[*]} => target_minutes=( ${target_minutes_list[*]} ) => target_seconds=( ${target_seconds_list[*]} )"
### This code requires that the list of wav files to be generated is in ascending seconds order, i.e "120 300 900 1800)
if ! spawn_wav_recording_daemon ${receiver_name} ${receiver_band} ; then
local ret_code=$?
wd_logger 1 "ERROR: 'spawn_wav_recording_daemon ${receiver_name} ${receiver_band}' => ${ret_code}"
return ${ret_code}
fi
local raw_file_list=( $( find -maxdepth 1 \( -name \*.wav -o -name \*.raw \) | sed 's/\.\///' | sort ) ) ### minute-*.raw *_usb.wav) ### Get list of the one minute long 'raw' wav files being created by the Kiwi (.wav) or SDR ((.raw)
wd_logger 1 "Found ${#raw_file_list[@]} raw/wav files: '${raw_file_list[*]}'"
case ${#raw_file_list[@]} in
0 )
wd_logger 2 "There are no raw files. Wait up to 10 seconds for the first file to appear"
declare WAIT_FOR_FIRST_WAV_SECS=10
local timeout=0
while raw_file_list=( $( find -maxdepth 1 \( -name \*.wav -o -name \*.raw \) | sed 's/\.\///' | sort ) ) \
&& [[ ${#raw_file_list[@]} -eq 0 ]] \
&& [[ ${timeout} -lt ${WAIT_FOR_FIRST_WAV_SECS} ]]; do
sleep 1
(( ++timeout ))
done
if [[ ${#raw_file_list[@]} -eq 0 ]]; then
wd_logger 1 "Timeout after ${timeout} seconds while waiting for the first wav file to appear"
else
wd_logger 2 "First file appeared after waiting ${timeout} seconds"
fi
return 1 ### Signal to calling function to try again
;;
1 )
wd_logger 2 "There is only 1 raw file ${raw_file_list[0]} and all modes need at least 2 minutes. So wait for this file to be filled"
sleep_until_raw_file_is_full ${raw_file_list[0]}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "Error while waiting for the first wav file to fill, 'sleep_until_raw_file_is_full ${raw_file_list[0]}' => ${ret_code} "
fi
return 2
;;
* )
wd_logger 2 "Found ${#raw_file_list[@]} files, so we *may* have enough 1 minute wav files to make up a WSPR pkt. Wait until the last file is full, then proceed to process the list."
local second_from_file_name=${raw_file_list[0]:13:2}
if [[ 10#${second_from_file_name} -ne 0 ]]; then
wd_logger 2 "Raw file '${raw_file_list[0]}' name says the first file recording starts at second ${second_from_file_name}, not at second 0, so flushing it"
wd_rm ${raw_file_list[0]}
return 3
fi
sleep_until_raw_file_is_full ${raw_file_list[-1]}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: while waiting for the last of ${#raw_file_list[@]} wav files to fill, 'sleep_until_raw_file_is_full ${raw_file_list[-1]}' => ${ret_code}. Sleep 5 before resuming search"
wd_sleep 5
return 4
fi
;;
esac
wd_logger 2 "Found ${#raw_file_list[@]} full raw files. Fill return list with lists of those raw files which are part of each WSPR mode"
local clean_files_string
cleanup_wav_file_list clean_files_string "${raw_file_list[*]}"
local clean_file_list=( ${clean_files_string} )
if [[ ${#clean_file_list[@]} -ne ${#raw_file_list[@]} ]]; then
if [[ ${#clean_file_list[@]} -eq 0 ]]; then
wd_logger 1 "ERROR: clean_file_list[] has no files"
return 1
fi
if [[ ${#clean_file_list[@]} -lt 2 ]]; then
wd_logger 1 "ERROR: clean_file_list[]='${clean_file_list[*]}' has less than the minimum 2 packets needed for the smallest WSPR packet. So return error and try again to find a good list"
return 1
fi
raw_file_list=( ${clean_file_list[@]} )
wd_logger 1 "ERROR: After cleanup, raw_file_list[]='${raw_file_list[*]}' which is enough for a minimm sized WSPR packet"
fi
### We now have a list of two or more full size raw files
get_file_start_time_info epoch_of_first_raw_file minute_of_first_raw_file ${raw_file_list[0]}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: 'get_file_start_time_info epoch_of_first_raw_file minute_of_first_raw_file ${raw_file_list[0]}' => ${ret_code}"
return 4
fi
wd_logger 1 "The first raw file ${raw_file_list[0]} write time is at minute ${minute_of_first_raw_file}"
local index_of_first_file_which_needs_to_be_saved=${#raw_file_list[@]} ### Presume we will need to keep none of the raw files
local return_list=()
local seconds_in_wspr_pkt
for seconds_in_wspr_pkt in ${target_seconds_list[@]} ; do
local raw_files_in_wav_file_count=$((seconds_in_wspr_pkt / 60))
wd_logger 1 "Check to see if we can create a new ${seconds_in_wspr_pkt} seconds long wav file from ${raw_files_in_wav_file_count} raw files"
### Check to see if we have previously returned some of these files in a previous call to this function
shopt -s nullglob
local wav_raw_pkt_list=( *.wav.${seconds_in_wspr_pkt}-secs )
shopt -u nullglob
local index_of_first_unreported_raw_file
local index_of_last_unreported_file
if [[ ${#wav_raw_pkt_list[@]} -eq 0 ]]; then
wd_logger 2 "Found no wav_secs files for wspr pkts of this length, so there were no previously reported packets of this length. So find index of first raw file that would start a wav file of this many seconds"
local minute_of_first_raw_sample=$(( 10#${minute_of_first_raw_file}))
if [[ ${receiver_name} =~ "SDR" ]]; then
$(( --minute_of_first_raw_sample ))
if [[ ${minute_of_first_raw_sample} -lt 0 ]]; then
minute_of_first_raw_sample=59
fi
wd_logger 1 "Adjusted minute_of_first_raw_sample by 1 minute to ${minute_of_first_raw_sample} which compensates for the fact that sdrTest writes the last bytes of a wav file in the following minute of the first bytes"
fi
local first_minute_raw_wspr_pkt_index=$(( minute_of_first_raw_sample % raw_files_in_wav_file_count ))
index_of_first_unreported_raw_file=$(( (raw_files_in_wav_file_count - first_minute_raw_wspr_pkt_index) % raw_files_in_wav_file_count ))
wd_logger 2 "Raw_file ${raw_file_list[0]} of minute ${minute_of_first_raw_sample} is raw pkt #${first_minute_raw_wspr_pkt_index} of a ${seconds_in_wspr_pkt} second long wspr packet. So start of next wav_raw will be found at raw_file index ${index_of_first_unreported_raw_file}"
else
wd_logger 2 "Found that we previously returned ${#wav_raw_pkt_list[@]} wav files of this length"
if [[ ${#wav_raw_pkt_list[@]} -eq 1 ]]; then
wd_logger 2 "There is only one wav_raw pkt ${wav_raw_pkt_list[@]}, so leave it alone"
else
local flush_count=$(( ${#wav_raw_pkt_list[@]} - 1 ))
local inner_flush_list=( ${wav_raw_pkt_list[@]:0:${flush_count}} )
if [[ ${#inner_flush_list[*]} -gt 0 ]]; then
wd_logger 2 "Flushing ${#inner_flush_list[@]} files '${inner_flush_list[*]}' leaving only ${wav_raw_pkt_list[-1]}"
rm ${inner_flush_list[*]}
else
wd_logger 1 "ERROR: wav_raw_pkt_list[] has ${#wav_raw_pkt_list[@]} files, but inner_flush_list[] is empty"
fi
fi
local filename_of_latest_wav_raw=${wav_raw_pkt_list[-1]}
local epoch_of_latest_wav_raw_file
local minute_of_latest_wav_raw_file
get_file_start_time_info epoch_of_latest_wav_raw_file minute_of_latest_wav_raw_file ${filename_of_latest_wav_raw}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: 'get_file_start_time_info epoch_of_latest_wav_raw_file minute_of_latest_wav_raw_file ${filename_of_latest_wav_raw}' => ${ret_code}"
return 5
fi
local index_of_first_reported_raw_file=$(( ( epoch_of_latest_wav_raw_file - epoch_of_first_raw_file ) / 60 ))
index_of_first_unreported_raw_file=$(( index_of_first_reported_raw_file + raw_files_in_wav_file_count ))
wd_logger 2 "Latest wav_raw ${filename_of_latest_wav_raw} has epoch ${epoch_of_latest_wav_raw_file}. epoch_of_first_raw_file == ${epoch_of_first_raw_file}. So index_of_first_unreported_raw_file = ${index_of_first_unreported_raw_file}"
fi
if [[ ${index_of_first_unreported_raw_file} -ge ${#raw_file_list[@]} ]]; then
wd_logger 2 "The first first raw file of a wav_raw file is not yet in the list of minute_raw[] files. So continue to search for the next WSPR pkt length"
continue
fi
### The first file is present, now see if the last file is also present
index_of_last_raw_file_for_this_wav_file=$(( index_of_first_unreported_raw_file + raw_files_in_wav_file_count - 1))
if [[ ${index_of_last_raw_file_for_this_wav_file} -ge ${#raw_file_list[@]} ]]; then
### The last file isn't present
if [[ ${index_of_first_unreported_raw_file} -lt ${index_of_first_file_which_needs_to_be_saved} ]]; then
wd_logger 1 "For ${seconds_in_wspr_pkt} second packet, the first unreported file '${raw_file_list[${index_of_first_unreported_raw_file}]}' is at index ${index_of_first_unreported_raw_file}, so adjust the current index_of_first_file_which_needs_to_be_saved from ${index_of_first_file_which_needs_to_be_saved} down to that index"
index_of_first_file_which_needs_to_be_saved=${index_of_first_unreported_raw_file}
fi
wd_logger 2 "The first unreported ${seconds_in_wspr_pkt} seconds raw file is at index ${index_of_first_unreported_raw_file}, but the last raw file is not yet present, so we can't yet create a wav file. So continue to search for the next WSPR pkt length"
continue
fi
### There is a run of files which together form a wav file of this seconds in length
local this_seconds_files="${seconds_in_wspr_pkt}:${raw_file_list[*]:${index_of_first_unreported_raw_file}:${raw_files_in_wav_file_count} }"
local this_seconds_comma_separated_file=${this_seconds_files// /,}
return_list+=( ${this_seconds_comma_separated_file} )
wd_logger 2 "Added file list for ${seconds_in_wspr_pkt} second long wav file to return list from index [${index_of_first_unreported_raw_file}:${index_of_last_raw_file_for_this_wav_file}] => ${this_seconds_comma_separated_file}"
if [[ -z "${index_of_first_unreported_raw_file-}" ]]; then
wd_logger 1 "ERROR: variable 'index_of_first_unreported_raw_file' has not been set"
#continue
fi
if [[ ${raw_file_list[${index_of_first_unreported_raw_file}]-} ]]; then
wd_logger 1 "ERROR: array element ${raw_file_list}[${index_of_first_unreported_raw_file}] "
#continue
fi
local wav_list_returned_file=${raw_file_list[${index_of_first_unreported_raw_file}]}.${seconds_in_wspr_pkt}-secs
shopt -s nullglob
local flush_list=( *.${seconds_in_wspr_pkt}-secs )
shopt -u nullglob
if [[ ${#flush_list[@]} -gt 0 ]]; then
wd_logger 1 "For ${seconds_in_wspr_pkt} second packet, flushing ${#flush_list[@]} old wav_raw file(s): ${flush_list[*]}"
rm -f ${flush_list[@]} ### We only need to remember this new wav_raw file, so flush all older ones.
fi
if [[ ${seconds_in_wspr_pkt} == "120" ]]; then
local minute_of_first_unreported_raw_file=${wav_list_returned_file:11:2}
local decimal_minute=$(( 10#${minute_of_first_unreported_raw_file} % 2))
if [[ ${decimal_minute} -eq 0 ]]; then
wd_logger 1 "For 120 second wav file, returning an even minute start wav file '${wav_list_returned_file}'"
else
wd_logger 1 "ERROR: for 120 second wav file, returning an odd minute start wav file '${wav_list_returned_file}'"
fi
fi
touch -r ${raw_file_list[${index_of_first_unreported_raw_file}]} ${wav_list_returned_file}
if [[ ${index_of_first_unreported_raw_file} -lt ${index_of_first_file_which_needs_to_be_saved} ]]; then
wd_logger 1 "Added a new report list to be returned and remembering to save the files in it by changing the current index_of_first_file_which_needs_to_be_saved=${index_of_first_file_which_needs_to_be_saved} to index_of_first_unreported_raw_file=${index_of_first_unreported_raw_file}"
index_of_first_file_which_needs_to_be_saved=${index_of_first_unreported_raw_file}
fi
wd_logger 2 "For ${seconds_in_wspr_pkt} packet, remembered that a list for this wav file has been returned to the decoder by creating the zero length file ${wav_list_returned_file}"
done
if [[ ${index_of_first_file_which_needs_to_be_saved} -lt ${#raw_file_list[@]} ]] ; then
local count_of_raw_files_to_flush=$(( index_of_first_file_which_needs_to_be_saved ))
wd_logger 1 "After searching for all requested wav file lengths, found file [${index_of_first_file_which_needs_to_be_saved}] '${raw_file_list[${index_of_first_file_which_needs_to_be_saved}]}' is the oldest file which needs to be saved"
if [[ ${count_of_raw_files_to_flush} -gt 0 ]]; then
wd_logger 1 "So purging files '${raw_file_list[*]:0:${count_of_raw_files_to_flush}}'"
rm ${raw_file_list[@]:0:${count_of_raw_files_to_flush}}
fi
fi
wd_logger 2 "Returning ${#return_list[@]} wav file lists: '${return_list[*]}'"
eval ${return_variable_name}=\"${return_list[*]}\"
return 0
}
### Called by the decoding_daemon() to create an enhanced_spot file from the output of ALL_WSPR.TXT
### That enhanced_spot file is then posted to the subdirectory where the posting_daemon will process it (and other enhanced_spot files if this receiver is part of a MERGEd group)
### For future reference, here is the output lines in ALL_WSPR.TXT taken from the wsjt-x 2.1-2 source code:
# In WSJT-x v 2.2+, the wsprd decoder was enhanced. That new wsprd can be detected because it outputs 17 fields to each line of ALL_WSPR.TXT
# fprintf(fall_wspr, "%6s %4s %3.0f %5.2f %11.7f %-22s %2d %5.2f %2d %2d %4d %2d %3d %5u %5d \n",
# date, time, snr, dt, freq, message, (int)drift, sync, ipass+1, blocksize, jitter, decodetype, nhardmin, cycles/81, metric);
declare FIELD_COUNT_DECODE_LINE_WITH_GRID=18 ### wsprd v2.2 adds two fields and we have added the 'upload to wsprnet.org' field, so lines with a GRID will have 17 + 1 + 2 noise level fields. V3.x added spot_mode to the end of each line
declare FIELD_COUNT_DECODE_LINE_WITHOUT_GRID=$((FIELD_COUNT_DECODE_LINE_WITH_GRID - 1)) ### Lines without a GRID will have one fewer field
function create_enhanced_spots_file_and_queue_to_posting_daemon () {
local real_receiver_wspr_spots_file=$1 ### file with the new spot lines found in ALL_WSPR.TXT
local spot_file_date=$2 ### These are prepended to the output file name
local spot_file_time=$3
local wspr_cycle_rms_noise=$4 ### The following fields are the same for every spot in the wspr cycle
local wspr_cycle_fft_noise=$5
local wspr_cycle_kiwi_overloads_count=$6
local real_receiver_call_sign=$7 ### For real receivers, these are taken from the conf file line
local real_receiver_grid=$8 ### But for MERGEd receivers, the posting daemon will change them to the call+grid of the MERGEd receiver
local proxy_upload_this_spot=0 ### This is the last field of the enhanced_spot line. If ${SIGNAL_LEVEL_UPLOAD} == "proxy" AND this is the only spot (or best spot among a MERGEd group),
### then the posting daemon will modify this last field to '1' to signal to the upload_server to forward this spot to wsprnet.org
local cached_spots_file_name="${spot_file_date}_${spot_file_time}_spots.txt"
if grep -q "<...>" ${real_receiver_wspr_spots_file} ; then
grep -v "<...>" ${real_receiver_wspr_spots_file} > no_unknown_type3_spots.txt
wd_logger 1 "Posting 'no_unknown_type3_spots.txt' since found '<...>' calls in ${real_receiver_wspr_spots_file}"
real_receiver_wspr_spots_file=no_unknown_type3_spots.txt
fi
if [[ ${REMOVE_WD_DUP_SPOTS-yes} =~ [Yy][Ee][Ss] ]]; then
local spot_count=$(wc -l < ${real_receiver_wspr_spots_file} )
local tx_calls=$( awk '{print $6}' ${real_receiver_wspr_spots_file} | sort -u )
local tx_calls_list=( ${tx_calls} )
if [[ ${#tx_calls_list[@]} -eq ${spot_count} ]]; then
wd_logger 1 "Found no dup spots among the ${#tx_calls_list[@]} spots in ${real_receiver_wspr_spots_file}, so record all the spots"
else
local no_dups_spot_file=${real_receiver_wspr_spots_file}.nodups
> ${no_dups_spot_file}
wd_logger 1 "Found some dup spots in ${real_receiver_wspr_spots_file} since the spot_count=${spot_count} is greater than the number of calls #tx_calls_list[@]=${#tx_calls_list[@]} "
local tx_call
for tx_call in ${tx_calls_list[@]} ; do
grep "${tx_call}" ${real_receiver_wspr_spots_file} > spot_lines.txt
if [[ $(wc -l < spot_lines.txt) -eq 1 ]]; then
cat spot_lines.txt >> ${no_dups_spot_file}
else
sort -k 3,3n spot_lines.txt | tail -n 1 > add_spot_line.txt
wd_logger 1 "Found duplicate spot lines for tx_call=${tx_call}:\n$(< spot_lines.txt)\nSo adding only this spot line with the best SNR:\n$( < add_spot_line.txt)"
cat add_spot_line.txt >> ${no_dups_spot_file}
fi
done
sort -k 5,5n ${no_dups_spot_file} > no_dup_spots.txt
wd_logger 1 "Posting the newly created 'no_dup_spots.txt' which differs from ${real_receiver_wspr_spots_file}:\n$(diff ${real_receiver_wspr_spots_file} no_dup_spots.txt)"
real_receiver_wspr_spots_file=no_dup_spots.txt
fi
fi
wd_logger 2 "Enhance the spot lines from ALL_WSPR_TXT in ${real_receiver_wspr_spots_file} into ${cached_spots_file_name}"
> ${cached_spots_file_name} ### truncates or creates a zero length file
local spot_line
while read spot_line ; do
wd_logger 3 "Enhance line '${spot_line}'"
local spot_line_list=(${spot_line/,/})
local spot_line_list_count=${#spot_line_list[@]}
local spot_date spot_time spot_snr spot_dt spot_freq spot_call other_fields ### the order of the first fields in the spot lines created by decoding_daemon()
read spot_date spot_time spot_snr spot_dt spot_freq spot_call other_fields <<< "${spot_line/,/}"
local spot_grid spot_pwr spot_drift spot_sync_quality spot_ipass spot_blocksize spot_jitter spot_decodetype spot_nhardmin spot_cycles spot_metric spot_pkt_mode ### the order of the rest of the fields in the spot lines created by decoding_daemon()
if [[ ${spot_line_list_count} -eq ${FIELD_COUNT_DECODE_LINE_WITH_GRID} ]]; then
read spot_grid spot_pwr spot_drift spot_sync_quality spot_ipass spot_blocksize spot_jitter spot_decodetype spot_nhardmin spot_cycles spot_metric spot_pkt_mode <<< "${other_fields}" ### Most spot lines have a GRID
elif [[ ${spot_line_list_count} -eq ${FIELD_COUNT_DECODE_LINE_WITHOUT_GRID} ]]; then
spot_grid="none"
read spot_pwr spot_drift spot_sync_quality spot_ipass spot_blocksize spot_jitter spot_decodetype spot_nhardmin spot_cycles spot_metric spot_pkt_mode <<< "${other_fields}" ### Most spot lines have a GRID
else
### The decoding daemon formated a line we don't recognize
wd_logger 1 "INTERNAL ERROR: unexpected number of fields ${spot_line_list_count} rather than the expected ${FIELD_COUNT_DECODE_LINE_WITH_GRID} or ${FIELD_COUNT_DECODE_LINE_WITHOUT_GRID} in ALL_WSPR.TXT spot line '${spot_line}'"
continue
fi
### G3ZIL April 2020 V1 add azi to each spot line
wd_logger 2 "'add_derived ${spot_grid} ${real_receiver_grid} ${spot_freq}'"
add_derived ${spot_grid} ${real_receiver_grid} ${spot_freq}
if [[ ! -f ${DERIVED_ADDED_FILE} ]] ; then
wd_logger 2 "spots.txt ${DERIVED_ADDED_FILE} file not found"
return 1
fi
local derived_fields=$(cat ${DERIVED_ADDED_FILE} | tr -d '\r')
derived_fields=${derived_fields//,/ } ### Strip out the ,s
wd_logger 2 "derived_fields='${derived_fields}'"
local band km rx_az rx_lat rx_lon tx_az tx_lat tx_lon v_lat v_lon
read band km rx_az rx_lat rx_lon tx_az tx_lat tx_lon v_lat v_lon <<< "${derived_fields}"
if [[ ${spot_date} != ${spot_file_date} ]]; then
wd_logger 1 "WARNING: the date in spot line ${spot_date} doesn't match the date in the filename: ${spot_file_date}"
fi
if [[ ${spot_time} != ${spot_file_time} ]]; then
wd_logger 1 "WARNING: the time in spot line ${spot_time} doesn't match the time in the filename: ${spot_file_time}"
fi
### Output a space-separated line of enhanced spot data. The first 14 fields are in the same order but with "none" added when the message field with CALL doesn't include a GRID field
### Each of these lines should be uploaded to logs.wsprdaemon.org. If ${SIGNAL_LEVEL_UPLOAD} == "proxy" AND this is the only spot (or best spot among a MERGEd group), then the posting daemon will modify the last field to signal the upload_server to forward this spot to wsprnet.org
### The first row of printed variables are taken from the ALL_WSPR.TXT file lines with the 10th field sync_quality moved to field 3 so the line format is a superset of the lines created by WD 2.10
### The second row are the values added by our 'add_derived' Python line
### The third row are values taken from WD's rms_noise, fft_noise, WD.conf call sign and grid, etc.
# printf "%6s %4s %3.2f %3d %5.2f %12.7f %-14s %-6s %2d %2d %4d %4d %4d %4d %2d %3d %3d %2d %6.1f %6.1f %4d %6s %12s %5d %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %4d %4d\n" \
# field#: 1 2 10 3 4 5 6 7 8 9 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 \
printf "%6s %4s %3.2f %5.2f %5.2f %12.7f %-14s %-6s %2d %2d %4d %4d %4d %4d %2d %3d %3d %2d %6.1f %6.1f %4d %6s %12s %5d %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %4d %4d\n" \
${spot_date} ${spot_time} ${spot_sync_quality} ${spot_snr} ${spot_dt} ${spot_freq} ${spot_call} ${spot_grid} ${spot_pwr} ${spot_drift} ${spot_cycles} ${spot_jitter} ${spot_blocksize} ${spot_metric} ${spot_decodetype} ${spot_ipass} ${spot_nhardmin} ${spot_pkt_mode} ${wspr_cycle_rms_noise} ${wspr_cycle_fft_noise} ${band} ${real_receiver_grid} ${real_receiver_call_sign} ${km} ${rx_az} ${rx_lat} ${rx_lon} ${tx_az} ${tx_lat} ${tx_lon} ${v_lat} ${v_lon} ${wspr_cycle_kiwi_overloads_count} ${proxy_upload_this_spot} >> ${cached_spots_file_name}
local rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: printf \"%6s %4s %3.2f %3d %5.2f %12.7f %-14s %-6s %2d %2d %4d %4d %4d %4d %2d %3d %3d %2d %6.1f %6.1f %4d %6s %12s %5d %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %4d %4d\" \
'${spot_date}' '${spot_time}' '${spot_sync_quality}' '${spot_snr}' '${spot_dt}' '${spot_freq}' '${spot_call}' '${spot_grid}' '${spot_pwr}' '${spot_drift}' '${spot_cycles}' '${spot_jitter}' '${spot_blocksize}' '${spot_metric}' '${spot_decodetype}' \
'${spot_ipass}' '${spot_nhardmin}' '${spot_pkt_mode}' '${wspr_cycle_rms_noise}' '${wspr_cycle_fft_noise}' '${band}' '${real_receiver_grid}' '${real_receiver_call_sign}' '${km}' '${rx_az} ${rx_lat} ${rx_lon}' '${tx_az}' '${tx_lat}' '${tx_lon}' \
'${v_lat}' '${v_lon}' '${wspr_cycle_kiwi_overloads_count}' '${proxy_upload_this_spot}' => ${rc}"
fi
done < ${real_receiver_wspr_spots_file}
if [[ ! -s ${cached_spots_file_name} ]]; then
wd_logger 2 "Found no spots to queue, so queuing zero length spot file"
else
wd_logger 2 "Created '${cached_spots_file_name}' of size $(wc -c < ${cached_spots_file_name}):\n$(< ${cached_spots_file_name})"
fi
if grep "<...>" ${cached_spots_file_name} > bad_spots.txt; then
wd_logger 1 "Removing $(wc -l < bad_spots.txt) bad spot line(s) from upload:\n$(< bad_spots.txt)"
grep -v "<...>" ${cached_spots_file_name} > cleaned_spots.txt
mv cleaned_spots.txt ${cached_spots_file_name}
fi
### Queue the enhanced_spot file we have just created to all of the posting daemons
shopt -s nullglob ### * expands to NULL if there are no .wav wav_file
local dir
for dir in ${DECODING_CLIENTS_SUBDIR}/* ; do
### The decodes of this receiver/band are copied to one or more posting_subdirs where the posting_daemon will process them for posting to wsprnet.org
local decoding_client_spot_file_name=${dir}/${cached_spots_file_name}
if [[ -f ${decoding_client_spot_file_name} ]]; then
wd_logger 1 "ERROR: file ${decoding_client_spot_file_name} already exists, so dropping this new ${cached_spots_file_name}"
else
wd_logger 2 "Creating link from ${cached_spots_file_name} to ${decoding_client_spot_file_name} which is monitored by a posting daemon"
ln ${cached_spots_file_name} ${decoding_client_spot_file_name}
local rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: 'ln ${cached_spots_file_name} ${decoding_client_spot_file_name}' => ${rc}"
fi
fi
done
rm ${cached_spots_file_name} ### The links will persist until all the posting daemons delete them
wd_logger 2 "Done creating and queuing '${cached_spots_file_name}'"
}
function get_wsprdaemon_noise_queue_directory()
{
local __return_directory_name_return_variable=$1
local receiver_name=$2
local receiver_band=$3
local receiver_call_grid
receiver_call_grid=$( get_call_grid_from_receiver_name ${receiver_name} )
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: can't find receiver '${receiver_name}"
return 1
fi
### Linux directory names can't have the '/' character in them which is so common in ham callsigns. So replace all those '/' with '=' characters which (I am pretty sure) are never legal in call signs
local call_dir_name=${receiver_call_grid//\//=}
local noise_directory=${UPLOADS_WSPRDAEMON_NOISE_ROOT_DIR}/${receiver_call_grid}/${receiver_name}/${receiver_band}
mkdir -p ${noise_directory}
eval ${__return_directory_name_return_variable}=${noise_directory}
wd_logger 1 "Noise files from receiver_name=${receiver_name} receiver_band=${receiver_band} will be queued in ${noise_directory}"
return 0
}
function decoding_daemon() {
local receiver_name=$1 ### 'real' as opposed to 'merged' receiver
local receiver_band=${2}
local receiver_modes_arg=${3}
local receiver_call
receiver_call=$( get_receiver_call_from_name ${receiver_name} )
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: can't find receiver call from '${receiver_name}"
return 1
fi
local receiver_grid
receiver_grid=$( get_receiver_grid_from_name ${receiver_name} )
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: can't find receiver grid 'from ${receiver_name}"
return 1
fi
wd_logger 1 "Starting with args ${receiver_name} ${receiver_band} ${receiver_modes_arg}, receiver_call=${receiver_call} receiver_grid=${receiver_grid}"
View remainder of file in raw view
0