aux platform

cmd/extract.go

@@ -11,10 +11,10 @@ var extractCmd = &cobra.Command{
 	Long:  `Extract data from raw data files`,
 	Run: func(cmd *cobra.Command, args []string) {
 		params := extract.Params{
-			InputData:  "demo/4545.dat",
+			InputData:  "demo/051513.dat",
 			OutputPath: "demo/output",
 			TempPath:   "demo/temp",
-			DataId:     "004545",
+			DataId:     "051513",
 		}
 		p := extract.NewExtractor(&params)
 		p.ExtractAosData()

cmd/parse.go

@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"fmt"
+
 	"github.com/spf13/cobra"
 	"starwiz.cn/sjy01/preprocessing/extract"
 )
@@ -10,14 +12,17 @@ var parseCmd = &cobra.Command{
 	Short: "Parse the data",
 	Long:  `Parse the data`,
 	Run: func(cmd *cobra.Command, args []string) {
+
 		params := extract.Params{
-			InputData:  "demo/4545.dat",
+			InputData:  "demo/051513.dat",
 			OutputPath: "demo/output",
 			TempPath:   "demo/temp",
-			DataId:     "004545",
+			DataId:     "051513",
 		}
 		p := extract.NewExtractor(&params)
-		p.ParseAuxPlatform("demo/temp/004545_S44_AUX1.dat")
+		// p.ParseAuxPlatform("demo/output/051513_S46_AUX1.dat")
+		fmt.Println("Reference Time: 2000-01-01 12:00:00 UTC, seconds:", extract.Time2000UTCSec())
+		p.SeprateAuxAndImgData("demo/temp/051513_S46.dat")
 	},
 }
 

extract/aos.go

@@ -11,7 +11,7 @@ import (
 
 const (
 	AOSFrameLength    = 1024
-	AOSTempDataPrefix = "AOS_"
+	AOSTempDataPrefix = "AOS_Data_"
 )
 
 var AOSSyncWord = []byte{0x1A, 0xCF, 0xFC, 0x1D}
@@ -62,7 +62,7 @@ func (p *Extractor) ExtractAosData() error {
 		}
 
 		// 读取完整帧
-		if ldpcCheck(rawData[i:i+1024]) != nil {
+		if LDPCCheck(rawData[i:i+1024]) != nil {
 			errFrameCnt++
 		}
 
@@ -72,11 +72,11 @@ func (p *Extractor) ExtractAosData() error {
 	}
 
 	log.Println("valid AOS frame cnt:", validFrameCnt)
-	log.Println("err AOS frame cnt:", errFrameCnt)
+	log.Println("error AOS frame cnt:", errFrameCnt)
 
 	return nil
 }
 
-func ldpcCheck(frame []byte) error {
+func LDPCCheck(frame []byte) error {
 	return nil
 }

extract/aux.go

@@ -4,9 +4,12 @@ import (
 	"bufio"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
+	"time"
 
 	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
@@ -103,6 +106,11 @@ func (afh AuxFrameHead) CheckFrmHead() bool {
 	return false
 }
 
+type Record struct {
+	index   int
+	frmHead *AuxFrameHead
+}
+
 // 从传输帧文件中分离辅助数据，分别存储到辅助数据文件 _AUX.dat 和图像数据文件 _IMG_{波谱}.dat 中
 func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
 	// 打开传输帧文件 - 一次读入内存
@@ -114,42 +122,124 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
 
 	log.Info("seprate aux and img data from", sDataFile)
 
+	// 数据质量分析
+	fimg, _ := os.Create("demo/temp/aux_img.txt")
+	defer fimg.Close()
+	fimg.WriteString("字节数 		帧头流水号 		文件号 		帧头时间 	中心辅助数据前4字节(行33-36)\n")
+	var qmap []*Record
+	for i := 0; i < len(sData); {
+		// 解析帧
+		if i+24 > len(sData) {
+			log.Info("length of frame head is not engough for frame head")
+			break
+		}
+
+		if sData[i] == 0xD1 && sData[i+1] == 0x5B && sData[i+2] == 0xD1 && sData[i+3] == 0x5B {
+			log.Debug("find package head: 0xD15BD15B")
+		} else {
+			i++
+			// log.Println(i,"not find package head: 0xD15BD15B, skip 1 byte")
+			continue
+		}
+
+		afh := &AuxFrameHead{}
+		afh.Decode(sData[i : i+24])
+
+		if !afh.IsValidFrmHead {
+			log.Info("invalid frame head of original raw data", afh.FrmHead, "i =", i)
+			i += 1
+		} else {
+			r := &Record{index: i, frmHead: afh}
+			qmap = append(qmap, r)
+			i += 24
+
+			// fmt.Println("sn:", afh.SerialNo, "time:", afh.TimeSec)
+			// fmt.Println(time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), 0).String())
+
+			utcTime := binary.BigEndian.Uint32(sData[i+32 : i+36])
+			// fmt.Println("utc time of platform aux:", utcTime)
+			// fmt.Println(time.Unix(int64(utcTime), 0).String())
+
+			// fmt.Println("waveway:", sData[i+36])
+			t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), 0)
+			tAux := time.Unix(int64(utcTime), 0)
+			fimg.WriteString(fmt.Sprintf("%d 	%d 		%d 		%s 		%s\n",
+				i,
+				afh.SerialNo,
+				afh.FileNo,
+				t.String(),
+				tAux.String(),
+			))
+
+			// if len(qmap) > 32 {
+			// 	break
+			// }
+
+		}
+	}
+
+	sort.Slice(qmap, func(i, j int) bool {
+		return qmap[i].index < qmap[j].index
+	})
+
+	// lastPosOfPkg := 0
+	// lastLenOfPkg := 0
+	// for i, v := range qmap {
+	// 	pkgLen := v.index - lastPosOfPkg
+
+	// 	if v.index == 4112 {
+	// 		fmt.Println("index 4112's frm no", i+1)
+	// 		fmt.Println("last frm:", qmap[i-1].index)
+	// 		fmt.Println("next frm:", qmap[i+1].index)
+	// 	}
+
+	// 	if pkgLen != 23872 {
+	// 		log.Info("index:", v.index, "lastLenOfPkg:", lastLenOfPkg, "package length:", pkgLen)
+	// 	}
+
+	// 	lastLenOfPkg = pkgLen
+	// 	lastPosOfPkg = v.index
+	// }
+
+	return nil
+
 	name := filepath.Base(sDataFile)
 	name = strings.TrimRight(name, filepath.Ext(name))
+	outputDir := p.params.OutputPath
 
-	aux0 := filepath.Dir(sDataFile) + "/" + name + "_AUX0.dat"
+	aux0 := outputDir + "/" + name + "_EB.AUX"
 	os.Remove(aux0)
 	faux0, _ := os.OpenFile(aux0, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
 	waux0 := bufio.NewWriter(faux0)
 	defer waux0.Flush()
 	defer faux0.Close()
 
-	aux1 := filepath.Dir(sDataFile) + "/" + name + "_AUX1.dat"
+	aux1 := outputDir + "/" + name + "_PLAT.AUX"
 	os.Remove(aux1)
 	faux1, _ := os.OpenFile(aux1, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
 	waux1 := bufio.NewWriter(faux1)
 	defer waux1.Flush()
 	defer faux1.Close()
 
-	pan := filepath.Dir(sDataFile) + "/" + name + "_IMG_PAN.RAW"
+	pan := outputDir + "/" + name + "_IMG_PAN.RAW"
 	os.Remove(pan)
 	fpan, _ := os.OpenFile(pan, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
 	wpan := bufio.NewWriter(fpan)
 	defer wpan.Flush()
 	defer fpan.Close()
 	panEnviHdr := EnviHdr{}
-	wpanHdr, _ := NewBSQWriter(filepath.Dir(sDataFile)+"/"+name+"_IMG_PAN.HDR", &panEnviHdr)
+	wpanHdr, _ := NewBSQWriter(outputDir+"/"+name+"_IMG_PAN.HDR", &panEnviHdr)
 	defer wpanHdr.Close()
 
 	// 先按单波段存储，再按波段组合存储为 BSQ 格式的 MSS
-	mss := filepath.Dir(sDataFile) + "/" + name + "_IMG_MSS.RAW"
+	mss := outputDir + "/" + name + "_IMG_MSS.RAW"
 	os.Remove(mss)
 	fmss, _ := os.OpenFile(mss, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
 	wmss := bufio.NewWriter(fmss)
 	defer wmss.Flush()
 	defer fmss.Close()
 	mssEnviHdr := EnviHdr{}
-	wmssHdr, _ := NewBSQWriter(filepath.Dir(sDataFile)+"/"+name+"_IMG_MSS.HDR", &mssEnviHdr)
+	wmssHdr, _ := NewBSQWriter(outputDir+"/"+name+"_IMG_MSS.HDR", &mssEnviHdr)
 	defer wmssHdr.Close()
 
 	var afh AuxFrameHead
@@ -165,6 +255,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
 			log.Debug("find package head: 0xD15BD15B")
 		} else {
 			i++
+			// log.Println("not find package head: 0xD15BD15B, skip 1 byte")
 			continue
 		}
 
@@ -178,7 +269,8 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
 
 		if !afh.IsValidFrmHead {
 			log.Info("invalid frame head of original raw data")
-			break
+			i += 1
+			continue
 		}
 
 		// 目前只支持线阵模式

extract/aux_ebox.go

@@ -1,8 +1,10 @@
 package extract
 
 // 卫星时间起点 北京时间 2000-01-01 20:00:00
+var (
+	ReferenceTime2000 = 946728000
+)
 
 // 焦面电箱辅助数据 128 字节 (每 16 行原始图像数据为一组)
 type AuxFocalBox struct {
-
 }

extract/aux_platform.go

@@ -4,6 +4,7 @@ import (
 	"encoding/binary"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/sirupsen/logrus"
 )
@@ -13,8 +14,8 @@ const AuxPlatformFrmSize = 512 // 512 bytes
 // 中心机辅助数据 512 字节 (每 16 行原始图像数据为一组)
 type AuxPlatform struct {
 	// 结构中标注的字节序号从 1 开始计数
-	UTCTimeSec     uint32    // [1-4] 卫星 UTC 时间戳（秒
+	UTCTimeSec     uint32    // [1-4] 卫星 UTC 时间戳（秒）
-	Waveway        uint8     // [5] 波道
+	Waveway        uint8     // [5] 波道 0x00：波道1（241.0~311.0）0x01：波道2（241.1~311.1）
 	Microsecond    uint32    // [6-8]卫星秒小数（微秒）
 	QuatAttstarQ0  float64   // [9-12]定姿四元数(J2000) 的 Q0 值，量纲 1/0x40000000
 	QuatAttstarQ1  float64   // [13-16]Q1 值
@@ -147,12 +148,14 @@ func (ap *AuxPlatform) Parse(data []byte) error {
 func (ap AuxPlatform) Print() {
 	fmt.Println("--- aux frm ----")
 	fmt.Printf("UTCTimeSec: %d\n", ap.UTCTimeSec)
+	fmt.Println(time.Unix(int64(ap.UTCTimeSec), 0).String())
 	fmt.Printf("Microsecond: %d\n", ap.Microsecond)
 	fmt.Printf("DataTransLong: %f\n", ap.DataTransLong)
 	fmt.Printf("DataTransLat: %f\n", ap.DataTransLat)
 	fmt.Printf("DataTransH: %f\n", ap.DataTransH)
 	fmt.Println("--- ss1 ---")
 	fmt.Printf("SS1_UTCTime: %d\n", ap.SS1_UTCTime)
+	fmt.Println(time.Unix(int64(ap.SS1_UTCTime+uint32(ReferenceTime2000)), 0).String())
 	fmt.Printf("SS1_UTCTimeFrac: %f\n", ap.SS1_UTCTimeFrac)
 	fmt.Printf("SS1_ImgFrmNo: %d\n", ap.SS1_ImgFrmNo)
 	fmt.Println("--- ss2 ---")
@@ -188,3 +191,8 @@ func (e *Extractor) ParseAuxPlatform(auxfile string) ([]*AuxPlatform, error) {
 	}
 	return aps, nil
 }
+
+func Time2000UTCSec() int64 {
+	t, _ := time.ParseInLocation("2006-01-02 15:04:05", "2000-01-01 12:00:00", time.UTC)
+	return t.Unix()
+}

extract/params.go

@@ -9,4 +9,8 @@ type Params struct {
 	TempPath   string
 	Report     string
 	Result     string
+	Station    string
 }
+
+// L0 ID
+const L0_ID = `{{.Satellite}}_{{.Sensor}}_{{.YYMMDD}}_{{.HHMMSS}}_{{.DataId}}_{{.Index}}`

extract/trans_frame.go

@@ -50,6 +50,7 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) {
 	var sData []string
 	name := filepath.Base(p.params.InputData)
 	aosDataFile := filepath.Join(p.params.TempPath, AOSTempDataPrefix+name)
+	nullFrmCnt := 0
 
 	aosData, err := os.ReadFile(aosDataFile)
 	if err != nil {
@@ -60,6 +61,9 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) {
 	snRange := map[int][]uint32{}
 	datSet := map[int][]byte{}
 
+	fsn, _ := os.Create("demo/temp/tf_sn.txt")
+	defer fsn.Close()
+
 	var i int
 	for i < len(aosData) {
 		if i+4 > len(aosData) {
@@ -69,7 +73,7 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) {
 
 		if aosData[i] == 0xE7 && aosData[i+1] == 0x7E && aosData[i+2] == 0xE7 && aosData[i+3] == 0x7E {
 			if i+TransImageFrameLength > len(aosData) {
-				log.Println("trans frame length error")
+				log.Println("there are not enough data for a trans frame", len(aosData)-i)
 				break
 			}
 
@@ -77,10 +81,13 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) {
 			tf.Decode(aosData[i : i+TransImageFrameLength])
 			fileno := int(tf.FileNo)
 			snRange[fileno] = append(snRange[fileno], tf.SNo)
+			fsn.WriteString(fmt.Sprintf("%d 	%d 		%d\n", i, tf.SNo, fileno))
 
 			// 只保留非空帧
 			if tf.FrameFlag != 0x55 && fileno != 0 {
 				datSet[fileno] = append(datSet[fileno], tf.Data...)
+			} else {
+				nullFrmCnt++
 			}
 
 			i += TransImageFrameLength
@@ -89,6 +96,8 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) {
 		i++
 	}
 
+	fmt.Println("nullFrmCnt:", nullFrmCnt)
+
 	for k, v := range snRange {
 		vv := slice.Unique(v)
 		sort.Slice(vv, func(i, j int) bool { return vv[i] < vv[j] })