From 7e4efd83a10d4215b095faf3eb8339264256269c Mon Sep 17 00:00:00 2001 From: nuknal Date: Thu, 16 May 2024 10:14:32 +0800 Subject: [PATCH] aux platform --- cmd/extract.go | 4 +- cmd/parse.go | 11 +++-- extract/aos.go | 8 +-- extract/aux.go | 106 +++++++++++++++++++++++++++++++++++++--- extract/aux_ebox.go | 4 +- extract/aux_platform.go | 12 ++++- extract/params.go | 4 ++ extract/trans_frame.go | 11 ++++- 8 files changed, 140 insertions(+), 20 deletions(-) diff --git a/cmd/extract.go b/cmd/extract.go index 2cc6633..40110c3 100644 --- a/cmd/extract.go +++ b/cmd/extract.go @@ -11,10 +11,10 @@ var extractCmd = &cobra.Command{ Long: `Extract data from raw data files`, Run: func(cmd *cobra.Command, args []string) { params := extract.Params{ - InputData: "demo/4545.dat", + InputData: "demo/051513.dat", OutputPath: "demo/output", TempPath: "demo/temp", - DataId: "004545", + DataId: "051513", } p := extract.NewExtractor(¶ms) p.ExtractAosData() diff --git a/cmd/parse.go b/cmd/parse.go index 702a337..db23a87 100644 --- a/cmd/parse.go +++ b/cmd/parse.go @@ -1,6 +1,8 @@ package cmd import ( + "fmt" + "github.com/spf13/cobra" "starwiz.cn/sjy01/preprocessing/extract" ) @@ -10,14 +12,17 @@ var parseCmd = &cobra.Command{ Short: "Parse the data", Long: `Parse the data`, Run: func(cmd *cobra.Command, args []string) { + params := extract.Params{ - InputData: "demo/4545.dat", + InputData: "demo/051513.dat", OutputPath: "demo/output", TempPath: "demo/temp", - DataId: "004545", + DataId: "051513", } p := extract.NewExtractor(¶ms) - p.ParseAuxPlatform("demo/temp/004545_S44_AUX1.dat") + // p.ParseAuxPlatform("demo/output/051513_S46_AUX1.dat") + fmt.Println("Reference Time: 2000-01-01 12:00:00 UTC, seconds:", extract.Time2000UTCSec()) + p.SeprateAuxAndImgData("demo/temp/051513_S46.dat") }, } diff --git a/extract/aos.go b/extract/aos.go index 9eba64b..8c6f12c 100644 --- a/extract/aos.go +++ b/extract/aos.go @@ -11,7 +11,7 @@ import ( const ( AOSFrameLength = 1024 - AOSTempDataPrefix = "AOS_" + AOSTempDataPrefix = "AOS_Data_" ) var AOSSyncWord = []byte{0x1A, 0xCF, 0xFC, 0x1D} @@ -62,7 +62,7 @@ func (p *Extractor) ExtractAosData() error { } // 读取完整帧 - if ldpcCheck(rawData[i:i+1024]) != nil { + if LDPCCheck(rawData[i:i+1024]) != nil { errFrameCnt++ } @@ -72,11 +72,11 @@ func (p *Extractor) ExtractAosData() error { } log.Println("valid AOS frame cnt:", validFrameCnt) - log.Println("err AOS frame cnt:", errFrameCnt) + log.Println("error AOS frame cnt:", errFrameCnt) return nil } -func ldpcCheck(frame []byte) error { +func LDPCCheck(frame []byte) error { return nil } diff --git a/extract/aux.go b/extract/aux.go index 0b0dc36..e786a33 100644 --- a/extract/aux.go +++ b/extract/aux.go @@ -4,9 +4,12 @@ import ( "bufio" "encoding/binary" "errors" + "fmt" "os" "path/filepath" + "sort" "strings" + "time" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" @@ -103,6 +106,11 @@ func (afh AuxFrameHead) CheckFrmHead() bool { return false } +type Record struct { + index int + frmHead *AuxFrameHead +} + // 从传输帧文件中分离辅助数据，分别存储到辅助数据文件 _AUX.dat 和图像数据文件 _IMG_{波谱}.dat 中 func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error { // 打开传输帧文件 - 一次读入内存 @@ -114,42 +122,124 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error { log.Info("seprate aux and img data from", sDataFile) + // 数据质量分析 + fimg, _ := os.Create("demo/temp/aux_img.txt") + defer fimg.Close() + fimg.WriteString("字节数 帧头流水号 文件号 帧头时间 中心辅助数据前4字节(行33-36)\n") + var qmap []*Record + for i := 0; i < len(sData); { + // 解析帧 + if i+24 > len(sData) { + log.Info("length of frame head is not engough for frame head") + break + } + + if sData[i] == 0xD1 && sData[i+1] == 0x5B && sData[i+2] == 0xD1 && sData[i+3] == 0x5B { + log.Debug("find package head: 0xD15BD15B") + } else { + i++ + // log.Println(i,"not find package head: 0xD15BD15B, skip 1 byte") + continue + } + + afh := &AuxFrameHead{} + afh.Decode(sData[i : i+24]) + + if !afh.IsValidFrmHead { + log.Info("invalid frame head of original raw data", afh.FrmHead, "i =", i) + i += 1 + } else { + r := &Record{index: i, frmHead: afh} + qmap = append(qmap, r) + i += 24 + + // fmt.Println("sn:", afh.SerialNo, "time:", afh.TimeSec) + // fmt.Println(time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), 0).String()) + + utcTime := binary.BigEndian.Uint32(sData[i+32 : i+36]) + // fmt.Println("utc time of platform aux:", utcTime) + // fmt.Println(time.Unix(int64(utcTime), 0).String()) + + // fmt.Println("waveway:", sData[i+36]) + t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), 0) + tAux := time.Unix(int64(utcTime), 0) + fimg.WriteString(fmt.Sprintf("%d %d %d %s %s\n", + i, + afh.SerialNo, + afh.FileNo, + t.String(), + tAux.String(), + )) + + // if len(qmap) > 32 { + // break + // } + + } + } + + sort.Slice(qmap, func(i, j int) bool { + return qmap[i].index < qmap[j].index + }) + + // lastPosOfPkg := 0 + // lastLenOfPkg := 0 + // for i, v := range qmap { + // pkgLen := v.index - lastPosOfPkg + + // if v.index == 4112 { + // fmt.Println("index 4112's frm no", i+1) + // fmt.Println("last frm:", qmap[i-1].index) + // fmt.Println("next frm:", qmap[i+1].index) + // } + + // if pkgLen != 23872 { + // log.Info("index:", v.index, "lastLenOfPkg:", lastLenOfPkg, "package length:", pkgLen) + // } + + // lastLenOfPkg = pkgLen + // lastPosOfPkg = v.index + // } + + return nil + name := filepath.Base(sDataFile) name = strings.TrimRight(name, filepath.Ext(name)) + outputDir := p.params.OutputPath - aux0 := filepath.Dir(sDataFile) + "/" + name + "_AUX0.dat" + aux0 := outputDir + "/" + name + "_EB.AUX" os.Remove(aux0) faux0, _ := os.OpenFile(aux0, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777) waux0 := bufio.NewWriter(faux0) defer waux0.Flush() defer faux0.Close() - aux1 := filepath.Dir(sDataFile) + "/" + name + "_AUX1.dat" + aux1 := outputDir + "/" + name + "_PLAT.AUX" os.Remove(aux1) faux1, _ := os.OpenFile(aux1, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777) waux1 := bufio.NewWriter(faux1) defer waux1.Flush() defer faux1.Close() - pan := filepath.Dir(sDataFile) + "/" + name + "_IMG_PAN.RAW" + pan := outputDir + "/" + name + "_IMG_PAN.RAW" os.Remove(pan) fpan, _ := os.OpenFile(pan, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777) wpan := bufio.NewWriter(fpan) defer wpan.Flush() defer fpan.Close() panEnviHdr := EnviHdr{} - wpanHdr, _ := NewBSQWriter(filepath.Dir(sDataFile)+"/"+name+"_IMG_PAN.HDR", &panEnviHdr) + wpanHdr, _ := NewBSQWriter(outputDir+"/"+name+"_IMG_PAN.HDR", &panEnviHdr) defer wpanHdr.Close() // 先按单波段存储，再按波段组合存储为 BSQ 格式的 MSS - mss := filepath.Dir(sDataFile) + "/" + name + "_IMG_MSS.RAW" + mss := outputDir + "/" + name + "_IMG_MSS.RAW" os.Remove(mss) fmss, _ := os.OpenFile(mss, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777) wmss := bufio.NewWriter(fmss) defer wmss.Flush() defer fmss.Close() mssEnviHdr := EnviHdr{} - wmssHdr, _ := NewBSQWriter(filepath.Dir(sDataFile)+"/"+name+"_IMG_MSS.HDR", &mssEnviHdr) + wmssHdr, _ := NewBSQWriter(outputDir+"/"+name+"_IMG_MSS.HDR", &mssEnviHdr) defer wmssHdr.Close() var afh AuxFrameHead @@ -165,6 +255,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error { log.Debug("find package head: 0xD15BD15B") } else { i++ + // log.Println("not find package head: 0xD15BD15B, skip 1 byte") continue } @@ -178,7 +269,8 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error { if !afh.IsValidFrmHead { log.Info("invalid frame head of original raw data") - break + i += 1 + continue } // 目前只支持线阵模式 diff --git a/extract/aux_ebox.go b/extract/aux_ebox.go index a5ea9e4..489607f 100644 --- a/extract/aux_ebox.go +++ b/extract/aux_ebox.go @@ -1,8 +1,10 @@ package extract // 卫星时间起点 北京时间 2000-01-01 20:00:00 +var ( + ReferenceTime2000 = 946728000 +) // 焦面电箱辅助数据 128 字节 (每 16 行原始图像数据为一组) type AuxFocalBox struct { - } diff --git a/extract/aux_platform.go b/extract/aux_platform.go index d72d1b6..991be3f 100644 --- a/extract/aux_platform.go +++ b/extract/aux_platform.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "fmt" "os" + "time" "github.com/sirupsen/logrus" ) @@ -13,8 +14,8 @@ const AuxPlatformFrmSize = 512 // 512 bytes // 中心机辅助数据 512 字节 (每 16 行原始图像数据为一组) type AuxPlatform struct { // 结构中标注的字节序号从 1 开始计数 - UTCTimeSec uint32 // [1-4] 卫星 UTC 时间戳（秒 - Waveway uint8 // [5] 波道 + UTCTimeSec uint32 // [1-4] 卫星 UTC 时间戳（秒） + Waveway uint8 // [5] 波道 0x00：波道1（241.0~311.0）0x01：波道2（241.1~311.1） Microsecond uint32 // [6-8]卫星秒小数（微秒） QuatAttstarQ0 float64 // [9-12]定姿四元数(J2000) 的 Q0 值，量纲 1/0x40000000 QuatAttstarQ1 float64 // [13-16]Q1 值 @@ -147,12 +148,14 @@ func (ap *AuxPlatform) Parse(data []byte) error { func (ap AuxPlatform) Print() { fmt.Println("--- aux frm ----") fmt.Printf("UTCTimeSec: %d\n", ap.UTCTimeSec) + fmt.Println(time.Unix(int64(ap.UTCTimeSec), 0).String()) fmt.Printf("Microsecond: %d\n", ap.Microsecond) fmt.Printf("DataTransLong: %f\n", ap.DataTransLong) fmt.Printf("DataTransLat: %f\n", ap.DataTransLat) fmt.Printf("DataTransH: %f\n", ap.DataTransH) fmt.Println("--- ss1 ---") fmt.Printf("SS1_UTCTime: %d\n", ap.SS1_UTCTime) + fmt.Println(time.Unix(int64(ap.SS1_UTCTime+uint32(ReferenceTime2000)), 0).String()) fmt.Printf("SS1_UTCTimeFrac: %f\n", ap.SS1_UTCTimeFrac) fmt.Printf("SS1_ImgFrmNo: %d\n", ap.SS1_ImgFrmNo) fmt.Println("--- ss2 ---") @@ -188,3 +191,8 @@ func (e *Extractor) ParseAuxPlatform(auxfile string) ([]*AuxPlatform, error) { } return aps, nil } + +func Time2000UTCSec() int64 { + t, _ := time.ParseInLocation("2006-01-02 15:04:05", "2000-01-01 12:00:00", time.UTC) + return t.Unix() +} diff --git a/extract/params.go b/extract/params.go index 6bfa041..5405d68 100644 --- a/extract/params.go +++ b/extract/params.go @@ -9,4 +9,8 @@ type Params struct { TempPath string Report string Result string + Station string } + +// L0 ID +const L0_ID = `{{.Satellite}}_{{.Sensor}}_{{.YYMMDD}}_{{.HHMMSS}}_{{.DataId}}_{{.Index}}` diff --git a/extract/trans_frame.go b/extract/trans_frame.go index d742e84..1923856 100644 --- a/extract/trans_frame.go +++ b/extract/trans_frame.go @@ -50,6 +50,7 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) { var sData []string name := filepath.Base(p.params.InputData) aosDataFile := filepath.Join(p.params.TempPath, AOSTempDataPrefix+name) + nullFrmCnt := 0 aosData, err := os.ReadFile(aosDataFile) if err != nil { @@ -60,6 +61,9 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) { snRange := map[int][]uint32{} datSet := map[int][]byte{} + fsn, _ := os.Create("demo/temp/tf_sn.txt") + defer fsn.Close() + var i int for i < len(aosData) { if i+4 > len(aosData) { @@ -69,7 +73,7 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) { if aosData[i] == 0xE7 && aosData[i+1] == 0x7E && aosData[i+2] == 0xE7 && aosData[i+3] == 0x7E { if i+TransImageFrameLength > len(aosData) { - log.Println("trans frame length error") + log.Println("there are not enough data for a trans frame", len(aosData)-i) break } @@ -77,10 +81,13 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) { tf.Decode(aosData[i : i+TransImageFrameLength]) fileno := int(tf.FileNo) snRange[fileno] = append(snRange[fileno], tf.SNo) + fsn.WriteString(fmt.Sprintf("%d %d %d\n", i, tf.SNo, fileno)) // 只保留非空帧 if tf.FrameFlag != 0x55 && fileno != 0 { datSet[fileno] = append(datSet[fileno], tf.Data...) + } else { + nullFrmCnt++ } i += TransImageFrameLength @@ -89,6 +96,8 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) { i++ } + fmt.Println("nullFrmCnt:", nullFrmCnt) + for k, v := range snRange { vv := slice.Unique(v) sort.Slice(vv, func(i, j int) bool { return vv[i] < vv[j] })