#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Thu Jan 20 10:55:22 2022 @author: West Zhican Chen """ import sys import os import numpy as np import scipy.io as sio import collections from optparse import OptionParser import h5py import time import matplotlib.pyplot as plt from mpl_toolkits import mplot3d from matplotlib import ticker, cm import pickle from channel_analysis import * import hdf5_lib from hdf5_lib import * from hdf5process import * ##################################################################################### # # DATA PROCESSING # ##################################################################################### def cal_trace(hdf5path, chanpath = None, hdf5data = None, n_frames_to_inspect = 300, n_fr_insp_st = 100, sub_sample = 1, analysis_power = False, badTXnodes = None, badRXnodes = None, tx_pol = None, rx_pol = None, analysis_pmap = False, figs_path = None): """ Recursively calculate the channel value from hdf5 data Input: ------ hdf5path: str, path to all the hdf5 data chanpath: str, path to save the calculated channel value, only useful when not None hdf5data: str, if given then script will process the single given hdf5 data, o.t.w. will process the whole hdf5 files in hdf5path recursively n_frames_to_inspect: int, number of frames to inspect n_fr_insp_st: int, index of starting frame sub_sample: int, subsample incremental analysis_power: bool, whether to analyze the calculated trace badTXnodes: list of int, bad TX nodes to dismiss from the trace analysis badRXnodes: list of int, bad RX nodes to dismiss from the trace analysis tx_pol: int from (0, 1), selected polarization of the tx antennas rx_pol: int from (0, 1), selected polarization of the rx antennas analysis_pmap: bool, whether to analyze the correlation peak map of calculated trace figs_path: str, path to parent folder to save the figures Output: ------ None """ print("Inspecting hdf5 data from: {}".format(hdf5path)) if hdf5data: print("Inspecting the following hdf5 data:\n{}".format(hdf5data)) hdf5list = [] hdf5list.append(hdf5data) else: print("hdf5 data not specified, will process all the available hdf5 data recursively!") hdf5l = os.listdir(hdf5path) hdf5list = [ele for ele in hdf5l if ele[-4:] == 'hdf5'] # calculate channel matrix and save to file for hdf5file in hdf5list: hdf5 = hdf5_lib(hdf5path + hdf5file, n_frames_to_inspect = n_frames_to_inspect, n_fr_insp_st = n_fr_insp_st, sub_sample = sub_sample) data_pilot, data_noise = process_hdf5(hdf5) chan_full = data_pilot['chan_full'] chan_sci = data_pilot['chan_sci'] # saving the calculated channel value to chanpath if chanpath: chandict = {'chan_full': chan_full, 'chan_sci': chan_sci} chanfile = hdf5file[:-5] sio.savemat(chanpath + chanfile + '.mat', chandict) # with open(chanpath + chanfile, 'wb') as chanf: # pickle.dump(chandict, chanf) if analysis_power: # analysis the power numTXnodes, numRXnodes = chan_sci.shape fullTXnodes, fullRXnodes = list(np.arange(numTXnodes)), list(np.arange(numRXnodes)) goodTXnodes = [node for node in fullTXnodes if node not in badTXnodes] if badTXnodes else fullTXnodes goodRXnodes = [node for node in fullRXnodes if node not in badRXnodes] if badTXnodes else fullRXnodes chan_plot = chan_sci[goodTXnodes, :][:, goodRXnodes] if tx_pol != -1: chan_plot = chan_plot[tx_pol::2, :] if rx_pol != -1: chan_plot = chan_plot[:, rx_pol::2] power_chan_sci_dB = chan2power(chan_plot, plt = True) if analysis_pmap: # recursively calculate the peak percentage map from hdf5 data # make directory for this hdf5 file hdf5figp = hdf5file[:-5] + '/' fig_path = os.path.join(figs_path, hdf5figp) os.makedirs(fig_path, exist_ok=True) seq_found = hdf52pmap(data_pilot, hdf5.metadata, n_frm_st = hdf5.n_frm_st) pmap2pic(seq_found, n_frm_st = hdf5.n_frm_st, n_frm_end = hdf5.n_frm_end, fig_path = fig_path) def save_trace_from_fmap(hdf5path, chanpath, pmapsrc, hdf5data = None, sub_sample = 1, max_frames = 100): """ recursively call cal_trace with pre-set index (based on the precalculated peak map) and save the calculated trace Input: ------ hdf5path: str, path to all the hdf5 data chanpath: str, path to save the calculated channel value pmapsrc: str, filename of the correlation peak map hdf5data: str, if given then script will process the single given hdf5 data, o.t.w. will process the whole hdf5 files in hdf5path recursively sub_sample: int, subsample incremental max_frames: int, maximum number of frames to use Output: ------ NA """ # load the correlation peak map pmap = np.loadtxt(pmapsrc, delimiter = ',', dtype = 'str') print("Inspecting hdf5 data from: {}".format(hdf5path)) if hdf5data: print("Inspecting the following hdf5 data:\n{}".format(hdf5data)) hdf5list = [] hdf5list.append(hdf5data) else: print("hdf5 data not specified, will process all the available hdf5 data recursively!") hdf5l = os.listdir(hdf5path) hdf5list = [ele for ele in hdf5l if ele[-4:] == 'hdf5'] for hdf5file in hdf5list: for idx in range(pmap.shape[0]): # simple and empirical pattern matching # will most likely fail for more general purpose tasks if hdf5file[-9:] == pmap[idx, 0][-9:]: hdf5idx = idx n_fr_insp_st = int(pmap[hdf5idx, 1]) n_frames_to_inspect = min(int(pmap[hdf5idx, 2]) - n_fr_insp_st, max_frames) cal_trace(hdf5path, chanpath = chanpath, hdf5data = hdf5file, n_frames_to_inspect = n_frames_to_inspect, n_fr_insp_st = n_fr_insp_st, sub_sample = sub_sample) print("Successfully saved {} into {}".format(hdf5file, chanpath)) def main(): parser = OptionParser() parser.add_option("--hdf5-path", type="str", dest="hdf5path", help="path to hdf5 file", default="/space/west/RENEWLab/CC/Sounder/logs/") parser.add_option("--chan-path", type="str", dest="chanpath", help="path to save the calculated channel value, only useful when not None", default=None) parser.add_option("--hdf5-data", type="str", dest="hdf5data", help="if given then script will process the single given hdf5 file, o.t.w. will process the whole hdf5 files in hdf5path recursively", default=None) parser.add_option("--n-frames", type="int", dest="n_frames_to_inspect", help="Number of frames to inspect", default=300) parser.add_option("--frame-start", type="int", dest="n_fr_insp_st", help="Starting frame. Must have set n_frames first and make sure fr_strt is within boundaries ", default=0) parser.add_option("--sub-sample", type="int", dest="sub_sample", help="Sub sample rate", default=1) parser.add_option("--analysis-power", action="store_true", dest="analysis_power", help="analyze the power of the calculated trace", default=False) parser.add_option("--bad-TX", type="str", dest="badTXnodes", help="bad TX nodes to dismiss from the trace analysis, separated with ,", default=None) parser.add_option("--bad-RX", type="str", dest="badRXnodes", help="bad RX nodes to dismiss from the trace analysis, separated with ,", default=None) parser.add_option("--tx-pol", type="int", dest="tx_pol", help="int from (0, 1), selected polarization of the tx antennas", default=-1) parser.add_option("--rx-pol", type="int", dest="rx_pol", help="int from (0, 1), selected polarization of the tx antennas", default=-1) parser.add_option("--figs-path", type="str", dest="figs_path", help="path to the parent folder where correlation peak maps are stored", default=None) parser.add_option("--analysis-pmap", action="store_true", dest="analysis_pmap", help="analyze the peak correlation map of the calculated trace", default=False) parser.add_option("--pmap-src", type="str", dest="pmapsrc", help="pmap source", default=None) parser.add_option("--max-frames", type="int", dest="max_frames", help="max number of frames to inspect", default=100) (options, args) = parser.parse_args() hdf5path = options.hdf5path chanpath = options.chanpath hdf5data = options.hdf5data n_frames_to_inspect = options.n_frames_to_inspect n_fr_insp_st = options.n_fr_insp_st sub_sample = options.sub_sample analysis_power = options.analysis_power if options.badTXnodes: badTXnodes = [int(item) for item in options.badTXnodes.split(',')] else: badTXnodes = None if options.badRXnodes: badRXnodes = [int(item) for item in options.badRXnodes.split(',')] else: badRXnodes = None tx_pol = options.tx_pol rx_pol = options.rx_pol figs_path = options.figs_path analysis_pmap = options.analysis_pmap pmapsrc = options.pmapsrc max_frames = options.max_frames if pmapsrc: assert chanpath, "chanpath needs to be specified with --pmap-src mode" save_trace_from_fmap(hdf5path, chanpath, pmapsrc, hdf5data, sub_sample, max_frames) return None # calculate the trace cal_trace(hdf5path, chanpath = chanpath, hdf5data = hdf5data, n_frames_to_inspect = n_frames_to_inspect, n_fr_insp_st = n_fr_insp_st, sub_sample = sub_sample, analysis_power = analysis_power, badTXnodes = badTXnodes, badRXnodes = badRXnodes, tx_pol = tx_pol, rx_pol = rx_pol, analysis_pmap = analysis_pmap, figs_path = figs_path) if __name__ == '__main__': main()