lankun/sjy01-preprocessing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

aux platform

Browse Source
This commit is contained in:
nuknal
2024-05-16 10:14:32 +08:00
parent 6a22735d78
commit 7e4efd83a1
8 changed files with 140 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/extract.go
View File

@@ -11,10 +11,10 @@ var extractCmd = &cobra.Command{
Long: `Extract data from raw data files`,
Run: func(cmd *cobra.Command, args []string) {
params := extract.Params{
InputData: "demo/4545.dat",
InputData: "demo/051513.dat",
OutputPath: "demo/output",
TempPath: "demo/temp",
DataId: "004545",
DataId: "051513",
}
p := extract.NewExtractor(&params)
p.ExtractAosData()

11
cmd/parse.go
View File

@@ -1,6 +1,8 @@
package cmd
import (
"fmt"
"github.com/spf13/cobra"
"starwiz.cn/sjy01/preprocessing/extract"
)
@@ -10,14 +12,17 @@ var parseCmd = &cobra.Command{
Short: "Parse the data",
Long: `Parse the data`,
Run: func(cmd *cobra.Command, args []string) {
params := extract.Params{
InputData: "demo/4545.dat",
InputData: "demo/051513.dat",
OutputPath: "demo/output",
TempPath: "demo/temp",
DataId: "004545",
DataId: "051513",
}
p := extract.NewExtractor(&params)
p.ParseAuxPlatform("demo/temp/004545_S44_AUX1.dat")
// p.ParseAuxPlatform("demo/output/051513_S46_AUX1.dat")
fmt.Println("Reference Time: 2000-01-01 12:00:00 UTC, seconds:", extract.Time2000UTCSec())
p.SeprateAuxAndImgData("demo/temp/051513_S46.dat")
},
}

8
extract/aos.go
View File

@@ -11,7 +11,7 @@ import (
const (
AOSFrameLength = 1024
AOSTempDataPrefix = "AOS_"
AOSTempDataPrefix = "AOS_Data_"
)
var AOSSyncWord = []byte{0x1A, 0xCF, 0xFC, 0x1D}
@@ -62,7 +62,7 @@ func (p *Extractor) ExtractAosData() error {
}
// 读取完整帧
if ldpcCheck(rawData[i:i+1024]) != nil {
if LDPCCheck(rawData[i:i+1024]) != nil {
errFrameCnt++
}
@@ -72,11 +72,11 @@ func (p *Extractor) ExtractAosData() error {
}
log.Println("valid AOS frame cnt:", validFrameCnt)
log.Println("err AOS frame cnt:", errFrameCnt)
log.Println("error AOS frame cnt:", errFrameCnt)
return nil
}
func ldpcCheck(frame []byte) error {
func LDPCCheck(frame []byte) error {
return nil
}

106
extract/aux.go
View File

@@ -4,9 +4,12 @@ import (
"bufio"
"encoding/binary"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"
@@ -103,6 +106,11 @@ func (afh AuxFrameHead) CheckFrmHead() bool {
return false
}
type Record struct {
index int
frmHead *AuxFrameHead
}
// 从传输帧文件中分离辅助数据，分别存储到辅助数据文件 _AUX.dat 和图像数据文件 _IMG_{波谱}.dat 中
func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
// 打开传输帧文件 - 一次读入内存
@@ -114,42 +122,124 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
log.Info("seprate aux and img data from", sDataFile)
// 数据质量分析
fimg, _ := os.Create("demo/temp/aux_img.txt")
defer fimg.Close()
fimg.WriteString("字节数 帧头流水号 文件号 帧头时间 中心辅助数据前4字节(行33-36)\n")
var qmap []*Record
for i := 0; i < len(sData); {
// 解析帧
if i+24 > len(sData) {
log.Info("length of frame head is not engough for frame head")
break
}
if sData[i] == 0xD1 && sData[i+1] == 0x5B && sData[i+2] == 0xD1 && sData[i+3] == 0x5B {
log.Debug("find package head: 0xD15BD15B")
} else {
i++
// log.Println(i,"not find package head: 0xD15BD15B, skip 1 byte")
continue
}
afh := &AuxFrameHead{}
afh.Decode(sData[i : i+24])
if !afh.IsValidFrmHead {
log.Info("invalid frame head of original raw data", afh.FrmHead, "i =", i)
i += 1
} else {
r := &Record{index: i, frmHead: afh}
qmap = append(qmap, r)
i += 24
// fmt.Println("sn:", afh.SerialNo, "time:", afh.TimeSec)
// fmt.Println(time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), 0).String())
utcTime := binary.BigEndian.Uint32(sData[i+32 : i+36])
// fmt.Println("utc time of platform aux:", utcTime)
// fmt.Println(time.Unix(int64(utcTime), 0).String())
// fmt.Println("waveway:", sData[i+36])
t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), 0)
tAux := time.Unix(int64(utcTime), 0)
fimg.WriteString(fmt.Sprintf("%d %d %d %s %s\n",
i,
afh.SerialNo,
afh.FileNo,
t.String(),
tAux.String(),
))
// if len(qmap) > 32 {
// break
// }
}
}
sort.Slice(qmap, func(i, j int) bool {
return qmap[i].index < qmap[j].index
})
// lastPosOfPkg := 0
// lastLenOfPkg := 0
// for i, v := range qmap {
// pkgLen := v.index - lastPosOfPkg
// if v.index == 4112 {
// fmt.Println("index 4112's frm no", i+1)
// fmt.Println("last frm:", qmap[i-1].index)
// fmt.Println("next frm:", qmap[i+1].index)
// }
// if pkgLen != 23872 {
// log.Info("index:", v.index, "lastLenOfPkg:", lastLenOfPkg, "package length:", pkgLen)
// }
// lastLenOfPkg = pkgLen
// lastPosOfPkg = v.index
// }
return nil
name := filepath.Base(sDataFile)
name = strings.TrimRight(name, filepath.Ext(name))
outputDir := p.params.OutputPath
aux0 := filepath.Dir(sDataFile) + "/" + name + "_AUX0.dat"
aux0 := outputDir + "/" + name + "_EB.AUX"
os.Remove(aux0)
faux0, _ := os.OpenFile(aux0, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
waux0 := bufio.NewWriter(faux0)
defer waux0.Flush()
defer faux0.Close()
aux1 := filepath.Dir(sDataFile) + "/" + name + "_AUX1.dat"
aux1 := outputDir + "/" + name + "_PLAT.AUX"
os.Remove(aux1)
faux1, _ := os.OpenFile(aux1, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
waux1 := bufio.NewWriter(faux1)
defer waux1.Flush()
defer faux1.Close()
pan := filepath.Dir(sDataFile) + "/" + name + "_IMG_PAN.RAW"
pan := outputDir + "/" + name + "_IMG_PAN.RAW"
os.Remove(pan)
fpan, _ := os.OpenFile(pan, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
wpan := bufio.NewWriter(fpan)
defer wpan.Flush()
defer fpan.Close()
panEnviHdr := EnviHdr{}
wpanHdr, _ := NewBSQWriter(filepath.Dir(sDataFile)+"/"+name+"_IMG_PAN.HDR", &panEnviHdr)
wpanHdr, _ := NewBSQWriter(outputDir+"/"+name+"_IMG_PAN.HDR", &panEnviHdr)
defer wpanHdr.Close()
// 先按单波段存储，再按波段组合存储为 BSQ 格式的 MSS
mss := filepath.Dir(sDataFile) + "/" + name + "_IMG_MSS.RAW"
mss := outputDir + "/" + name + "_IMG_MSS.RAW"
os.Remove(mss)
fmss, _ := os.OpenFile(mss, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
wmss := bufio.NewWriter(fmss)
defer wmss.Flush()
defer fmss.Close()
mssEnviHdr := EnviHdr{}
wmssHdr, _ := NewBSQWriter(filepath.Dir(sDataFile)+"/"+name+"_IMG_MSS.HDR", &mssEnviHdr)
wmssHdr, _ := NewBSQWriter(outputDir+"/"+name+"_IMG_MSS.HDR", &mssEnviHdr)
defer wmssHdr.Close()
var afh AuxFrameHead
@@ -165,6 +255,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
log.Debug("find package head: 0xD15BD15B")
} else {
i++
// log.Println("not find package head: 0xD15BD15B, skip 1 byte")
continue
}
@@ -178,7 +269,8 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
if !afh.IsValidFrmHead {
log.Info("invalid frame head of original raw data")
break
i += 1
continue
}
// 目前只支持线阵模式

4
extract/aux_ebox.go
View File

@@ -1,8 +1,10 @@
package extract
// 卫星时间起点 北京时间 2000-01-01 20:00:00
var (
ReferenceTime2000 = 946728000
)
// 焦面电箱辅助数据 128 字节 (每 16 行原始图像数据为一组)
type AuxFocalBox struct {
}

12
extract/aux_platform.go
View File

@@ -4,6 +4,7 @@ import (
"encoding/binary"
"fmt"
"os"
"time"
"github.com/sirupsen/logrus"
)
@@ -13,8 +14,8 @@ const AuxPlatformFrmSize = 512 // 512 bytes
// 中心机辅助数据 512 字节 (每 16 行原始图像数据为一组)
type AuxPlatform struct {
// 结构中标注的字节序号从 1 开始计数
UTCTimeSec uint32 // [1-4] 卫星 UTC 时间戳（秒
Waveway uint8 // [5] 波道
UTCTimeSec uint32 // [1-4] 卫星 UTC 时间戳（秒）
Waveway uint8 // [5] 波道 0x00：波道1（241.0~311.0）0x01：波道2（241.1~311.1）
Microsecond uint32 // [6-8]卫星秒小数（微秒）
QuatAttstarQ0 float64 // [9-12]定姿四元数(J2000) 的 Q0 值，量纲 1/0x40000000
QuatAttstarQ1 float64 // [13-16]Q1 值
@@ -147,12 +148,14 @@ func (ap *AuxPlatform) Parse(data []byte) error {
func (ap AuxPlatform) Print() {
fmt.Println("--- aux frm ----")
fmt.Printf("UTCTimeSec: %d\n", ap.UTCTimeSec)
fmt.Println(time.Unix(int64(ap.UTCTimeSec), 0).String())
fmt.Printf("Microsecond: %d\n", ap.Microsecond)
fmt.Printf("DataTransLong: %f\n", ap.DataTransLong)
fmt.Printf("DataTransLat: %f\n", ap.DataTransLat)
fmt.Printf("DataTransH: %f\n", ap.DataTransH)
fmt.Println("--- ss1 ---")
fmt.Printf("SS1_UTCTime: %d\n", ap.SS1_UTCTime)
fmt.Println(time.Unix(int64(ap.SS1_UTCTime+uint32(ReferenceTime2000)), 0).String())
fmt.Printf("SS1_UTCTimeFrac: %f\n", ap.SS1_UTCTimeFrac)
fmt.Printf("SS1_ImgFrmNo: %d\n", ap.SS1_ImgFrmNo)
fmt.Println("--- ss2 ---")
@@ -188,3 +191,8 @@ func (e *Extractor) ParseAuxPlatform(auxfile string) ([]*AuxPlatform, error) {
}
return aps, nil
}
func Time2000UTCSec() int64 {
t, _ := time.ParseInLocation("2006-01-02 15:04:05", "2000-01-01 12:00:00", time.UTC)
return t.Unix()
}

4
extract/params.go
View File

@@ -9,4 +9,8 @@ type Params struct {
TempPath string
Report string
Result string
Station string
}
// L0 ID
const L0_ID = `{{.Satellite}}_{{.Sensor}}_{{.YYMMDD}}_{{.HHMMSS}}_{{.DataId}}_{{.Index}}`

11
extract/trans_frame.go
View File

@@ -50,6 +50,7 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) {
var sData []string
name := filepath.Base(p.params.InputData)
aosDataFile := filepath.Join(p.params.TempPath, AOSTempDataPrefix+name)
nullFrmCnt := 0
aosData, err := os.ReadFile(aosDataFile)
if err != nil {
@@ -60,6 +61,9 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) {
snRange := map[int][]uint32{}
datSet := map[int][]byte{}
fsn, _ := os.Create("demo/temp/tf_sn.txt")
defer fsn.Close()
var i int
for i < len(aosData) {
if i+4 > len(aosData) {
@@ -69,7 +73,7 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) {
if aosData[i] == 0xE7 && aosData[i+1] == 0x7E && aosData[i+2] == 0xE7 && aosData[i+3] == 0x7E {
if i+TransImageFrameLength > len(aosData) {
log.Println("trans frame length error")
log.Println("there are not enough data for a trans frame", len(aosData)-i)
break
}
@@ -77,10 +81,13 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) {
tf.Decode(aosData[i : i+TransImageFrameLength])
fileno := int(tf.FileNo)
snRange[fileno] = append(snRange[fileno], tf.SNo)
fsn.WriteString(fmt.Sprintf("%d %d %d\n", i, tf.SNo, fileno))
// 只保留非空帧
if tf.FrameFlag != 0x55 && fileno != 0 {
datSet[fileno] = append(datSet[fileno], tf.Data...)
} else {
nullFrmCnt++
}
i += TransImageFrameLength
@@ -89,6 +96,8 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) {
i++
}
fmt.Println("nullFrmCnt:", nullFrmCnt)
for k, v := range snRange {
vv := slice.Unique(v)
sort.Slice(vv, func(i, j int) bool { return vv[i] < vv[j] })