diff --git a/cmd/extract.go b/cmd/extract.go index 8fcff86..93bc31e 100644 --- a/cmd/extract.go +++ b/cmd/extract.go @@ -3,14 +3,16 @@ package cmd import ( "fmt" + "github.com/k0kubun/pp/v3" "github.com/spf13/cobra" "starwiz.cn/sjy01/preprocessing/extract" ) var ( - dataId string - batch bool - output string + dataId string + batch bool + output string + paramsXML string ) var extractCmd = &cobra.Command{ @@ -18,6 +20,24 @@ var extractCmd = &cobra.Command{ Short: "Extract data from raw data files", Long: `Extract data from raw data files`, Run: func(cmd *cobra.Command, args []string) { + if paramsXML != "" { + params, err := extract.LoadL0Params(paramsXML) + if err != nil { + panic(err) + } + + pp.Println(params) + + e := extract.NewExtractor(params) + aos, _ := e.ExtractAosData() + dats, _ := e.ExtractOriginalImageData(aos) + for i, d := range dats { + e.SeprateAuxAndImgData(d, i) + } + e.Cleanup() + + return + } if batch { ps := params() @@ -55,6 +75,7 @@ func init() { extractCmd.Flags().StringVarP(&dataId, "data-id", "d", "051622", "051622") extractCmd.Flags().BoolVarP(&batch, "batch", "b", false, "true | false") extractCmd.Flags().StringVarP(&output, "out", "o", "demo/output", "demo/output") + extractCmd.Flags().StringVarP(¶msXML, "params", "x", "", "parameters file path") } func params() []*extract.Params { diff --git a/cmd/root.go b/cmd/root.go index 490fad3..cc3d885 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,10 +6,6 @@ import ( "github.com/spf13/cobra" ) -var ( - paramsXML string -) - var rootCmd = &cobra.Command{ Use: "SJY01 Preprocessing", Short: "Preprocessing tools for SJY01 original raw data", @@ -20,7 +16,7 @@ var rootCmd = &cobra.Command{ } func init() { - rootCmd.PersistentFlags().StringVarP(¶msXML, "params", "p", "params.xml", "parameters file path") + // rootCmd.PersistentFlags().StringVarP(¶msXML, "params", "p", "params.xml", "parameters file path") } func Execute() error { diff --git a/extract/aux_ebox.go b/extract/aux_ebox.go index 189981e..090db60 100644 --- a/extract/aux_ebox.go +++ b/extract/aux_ebox.go @@ -140,7 +140,7 @@ func (ab AuxFocalBox) SaveXlsx(f *excelize.File, col, row int) (int, error) { ab.TrainingDone, ab.WorkMode, ab.IntegralDirection, - ab.PGAGain, + ab.PGAGainValue(), ab.PIntegrationLevel, ab.B1IntegrationLevel, ab.B2IntegrationLevel, diff --git a/extract/aux_platform.go b/extract/aux_platform.go index 18a39aa..a356070 100644 --- a/extract/aux_platform.go +++ b/extract/aux_platform.go @@ -737,70 +737,6 @@ func (e *Extractor) ParseAuxPlatform(auxfile string) ([]*AuxPlatform, error) { return aps, nil } -func (e *Extractor) ParseAuxPlatformWithHead(auxfile string) ([]*AuxPlatform, error) { - data, err := os.ReadFile(auxfile) - if err != nil { - log.Println("read aux data from", auxfile, "error:", err.Error()) - return nil, err - } - - fimg, _ := os.Create("demo/temp/ref_051622_aux_img.txt") - defer fimg.Close() - fimg.WriteString("index 流水号 文件号 时间秒 秒小数 utcTime\n") - - var aps []*AuxPlatform - rows := 0 - for i := 0; i < len(data); { - if data[i] == 0xD1 && data[i+1] == 0x5B && data[i+2] == 0xD1 && data[i+3] == 0x5B { - log.Debug("find package head: 0xD15BD15B") - } else { - i++ - // log.Println(i,"not find package head: 0xD15BD15B, skip 1 byte") - continue - } - afh := &AuxFrameHead{} - afh.Decode(data[i : i+24]) - - if !afh.IsValidFrmHead { - log.Debugf("[%d] invalid frame head of original raw data %v", i, afh.FrmHead) - i += 1 - continue - } - - if (afh.SerialNo-1)%16 == 0 { - - ab := &AuxFocalBox{} - ab.Decode(data[i+24 : i+32]) - fmt.Println(ab.String()) - - utcTime := binary.BigEndian.Uint32(data[i+32 : i+36]) - - // t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), int64(afh.TimeSecFrac)*1000) - taux := time.Unix(int64(utcTime+uint32(ReferenceTime2000)), 0) - - fimg.WriteString( - fmt.Sprintf("%d %d %d %d %d %d %s\n", - i, - afh.SerialNo, - afh.FileNo, - afh.TimeSec, - afh.TimeSecFrac, - utcTime, - taux.String(), - )) - - rows++ - // if rows > 32 { - // break - // } - } - - i += 64 - - } - return aps, nil -} - func Time2000UTCSec() int64 { t, _ := time.ParseInLocation("2006-01-02 15:04:05", "2000-01-01 12:00:00", time.UTC) return t.Unix() diff --git a/extract/params.go b/extract/params.go index 5405d68..ed0d552 100644 --- a/extract/params.go +++ b/extract/params.go @@ -1,5 +1,11 @@ package extract +import ( + "encoding/xml" + "fmt" + "os" +) + type Params struct { InputData string OutputPath string @@ -12,5 +18,86 @@ type Params struct { Station string } +// 定义 InputFileList 结构体 +type InputFileList struct { + InputData string `xml:"inputdata"` +} + +// 定义 OutputFileList 结构体 +type OutputFileList struct { + Num int `xml:"num,attr"` + OutputPath string `xml:"outputPath"` + TempPath string `xml:"tempPath"` + ReportFile string `xml:"reportFile"` + ResultFile string `xml:"resultFile"` +} + +// 定义 Params 结构体 +type XMLParams struct { + StationId string `xml:"stationId"` + Sensor string `xml:"sensor"` + DataId string `xml:"dataId"` + SatelliteId string `xml:"satelliteId"` +} + +// 定义 Task 结构体 +type L0Task struct { + XMLName xml.Name `xml:"task"` + InputFileList InputFileList `xml:"inputFileList"` + OutputFileList OutputFileList `xml:"outputfilelist"` + Params XMLParams `xml:"params"` +} + // L0 ID const L0_ID = `{{.Satellite}}_{{.Sensor}}_{{.YYMMDD}}_{{.HHMMSS}}_{{.DataId}}_{{.Index}}` + +const L0TPLFile = "resource/template/satellite/sjy01.l0.xml.tpl" +const L0TPL = ` + + + {{.InputData}} + + + {{.OutputPath}} + {{.TempPath}} + {{.ReportFile}} + {{.ResultFile}} + + + WN + PMS + {{.DataId}} + SJY01 + + +` + +func LoadL0Params(fname string) (*Params, error) { + data, err := os.ReadFile(fname) + if err != nil { + return nil, err + } + + var task L0Task + err = xml.Unmarshal(data, &task) + if err != nil { + return nil, err + } + + var params Params + params.InputData = task.InputFileList.InputData + params.OutputPath = task.OutputFileList.OutputPath + params.Satellite = task.Params.SatelliteId + params.DataId = task.Params.DataId + params.Report = task.OutputFileList.ReportFile + params.Result = task.OutputFileList.ResultFile + params.Station = task.Params.StationId + + params.LogFile = fmt.Sprintf("/tmp/SJY01/log/%s_l0.log", params.DataId) + params.TempPath = task.OutputFileList.TempPath + if params.TempPath == "" { + params.TempPath = "/tmp/SJY01/data" + } + + return ¶ms, nil +} diff --git a/extract/process.go b/extract/process.go index cf3ef40..7686962 100644 --- a/extract/process.go +++ b/extract/process.go @@ -2,6 +2,7 @@ package extract import ( "os" + "path/filepath" ) type Extractor struct { @@ -10,8 +11,15 @@ type Extractor struct { } func NewExtractor(params *Params) *Extractor { - os.MkdirAll(params.OutputPath, 0755) - os.MkdirAll(params.TempPath, 0755) + if err := os.MkdirAll(params.OutputPath, 0755); err != nil { + panic(err) + } + if err := os.MkdirAll(params.TempPath, 0755); err != nil { + panic(err) + } + if err := os.MkdirAll(filepath.Base(params.LogFile), 0755); err != nil { + panic(err) + } return &Extractor{params: params, Clean: true} }