lankun/sjy01-preprocessing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

logger

Browse Source
This commit is contained in:
nuknal
2024-05-16 17:33:33 +08:00
parent 7e4efd83a1
commit 58acd444d6
10 changed files with 175 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
extract/aux.go
View File

@@ -123,14 +123,14 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
log.Info("seprate aux and img data from", sDataFile)
// 数据质量分析
fimg, _ := os.Create("demo/temp/aux_img.txt")
fimg, _ := os.Create("demo/temp/" + p.params.DataId + "_aux_img.txt")
defer fimg.Close()
fimg.WriteString("字节数 帧头流水号 文件号 帧头时间 中心辅助数据前4字节(行33-36)\n")
var qmap []*Record
for i := 0; i < len(sData); {
// 解析帧
if i+24 > len(sData) {
log.Info("length of frame head is not engough for frame head")
log.Info("length of original image frame head is not engough: ", len(sData)-i)
break
}
@@ -146,23 +146,24 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
afh.Decode(sData[i : i+24])
if !afh.IsValidFrmHead {
log.Info("invalid frame head of original raw data", afh.FrmHead, "i =", i)
log.Debugf("[%d] invalid frame head of original raw data %v", i, afh.FrmHead)
i += 1
} else {
r := &Record{index: i, frmHead: afh}
qmap = append(qmap, r)
i += 24
// fmt.Println("sn:", afh.SerialNo, "time:", afh.TimeSec)
// fmt.Println(time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), 0).String())
utcTime := binary.BigEndian.Uint32(sData[i+32 : i+36])
// fmt.Println("utc time of platform aux:", utcTime)
// fmt.Println(time.Unix(int64(utcTime), 0).String())
// fmt.Println("waveway:", sData[i+36])
t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), 0)
tAux := time.Unix(int64(utcTime), 0)
startAuxLine := tAux.Year() == t.Year() && tAux.Month() == t.Month() && tAux.Day() == t.Day() &&
tAux.Hour() == t.Hour() && tAux.Minute() == t.Minute()
if startAuxLine {
fimg.WriteString("----- AUX Start -----\n")
}
fimg.WriteString(fmt.Sprintf("%d %d %d %s %s\n",
i,
afh.SerialNo,
@@ -170,11 +171,6 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
t.String(),
tAux.String(),
))
// if len(qmap) > 32 {
// break
// }
}
}
@@ -182,27 +178,6 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
return qmap[i].index < qmap[j].index
})
// lastPosOfPkg := 0
// lastLenOfPkg := 0
// for i, v := range qmap {
// pkgLen := v.index - lastPosOfPkg
// if v.index == 4112 {
// fmt.Println("index 4112's frm no", i+1)
// fmt.Println("last frm:", qmap[i-1].index)
// fmt.Println("next frm:", qmap[i+1].index)
// }
// if pkgLen != 23872 {
// log.Info("index:", v.index, "lastLenOfPkg:", lastLenOfPkg, "package length:", pkgLen)
// }
// lastLenOfPkg = pkgLen
// lastPosOfPkg = v.index
// }
return nil
name := filepath.Base(sDataFile)
name = strings.TrimRight(name, filepath.Ext(name))
outputDir := p.params.OutputPath
@@ -244,6 +219,8 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
var afh AuxFrameHead
dataLen := len(sData)
mssData := make([][]byte, 4)
for i := 0; i < dataLen; {
if i+4 > dataLen {
logrus.Println("end of data, dataLen:", dataLen, "i:", i)
@@ -281,7 +258,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
// 读取一行数据 sData[i : i+afh.RowLength]
if i+afh.RowLength > dataLen {
log.Info("length of row data is not enough for row length:", afh.RowLength)
log.Infof("length of image row data %v is not enough: %v", dataLen-i, afh.RowLength)
break
}
@@ -300,31 +277,48 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
}
if afh.B1 {
wmss.Write(sData[dataIndex : dataIndex+1192])
// wmss.Write(sData[dataIndex : dataIndex+1192])
mssData[0] = append(mssData[0], sData[dataIndex:dataIndex+1192]...)
dataIndex += 1192
mssEnviHdr.Lines += 1
}
if afh.B2 {
wmss.Write(sData[dataIndex : dataIndex+1192])
// wmss.Write(sData[dataIndex : dataIndex+1192])
mssData[1] = append(mssData[1], sData[dataIndex:dataIndex+1192]...)
dataIndex += 1192
}
if afh.B3 {
wmss.Write(sData[dataIndex : dataIndex+1192])
// wmss.Write(sData[dataIndex : dataIndex+1192])
mssData[2] = append(mssData[2], sData[dataIndex:dataIndex+1192]...)
dataIndex += 1192
}
if afh.B4 {
wmss.Write(sData[dataIndex : dataIndex+1192])
// wmss.Write(sData[dataIndex : dataIndex+1192])
mssData[3] = append(mssData[3], sData[dataIndex:dataIndex+1192]...)
dataIndex += 1192
}
i = dataIndex // 完成一行数据解析
}
var bands = 0
for i := 0; i < 4; i++ {
if len(mssData[i]) > 0 {
log.Println("write mss data of band B", i+1)
_, err := wmss.Write(mssData[i])
if err != nil {
log.Error("write mss data error:", err.Error())
}
bands += 1
}
}
panEnviHdr.Samples = 9520
panEnviHdr.Bands = 1
wpanHdr.Write([]byte(panEnviHdr.String()))
mssEnviHdr.Lines = mssEnviHdr.Lines / 4 // 多光谱波段分别在 4 行中传输
mssEnviHdr.Bands = 1
mssEnviHdr.Samples = 2384 * 4
mssEnviHdr.Samples = 2384
mssEnviHdr.Bands = bands
wmssHdr.Write([]byte(mssEnviHdr.String()))
return nil

5
extract/aux_platform.go
View File

@@ -136,10 +136,10 @@ func (ap *AuxPlatform) Parse(data []byte) error {
ap.DataTransH = float64(binary.BigEndian.Uint16(data[213:215]))
ap.SS1_UTCTime = binary.BigEndian.Uint32(data[377:381])
ap.SS1_UTCTimeFrac = float32(binary.BigEndian.Uint16(data[381:384])) * 40.96
ap.SS1_UTCTimeFrac = float32(uint32(data[381])<<16|uint32(data[382])<<8|uint32(data[383])) * 40.96
ap.SS1_ImgFrmNo = uint32(data[410])<<16 | uint32(data[411])<<8 | uint32(data[412])
ap.SS2_UTCTime = binary.BigEndian.Uint32(data[423:427])
ap.SS2_UTCTimeFrac = float32(binary.BigEndian.Uint16(data[427:430])) * 40.96
ap.SS2_UTCTimeFrac = float32(uint32(data[427])<<16|uint32(data[428])<<8|uint32(data[429])) * 40.96
ap.SS1_ImgFrmNo = uint32(data[456])<<16 | uint32(data[457])<<8 | uint32(data[458])
return nil
@@ -180,7 +180,6 @@ func (e *Extractor) ParseAuxPlatform(auxfile string) ([]*AuxPlatform, error) {
return nil, err
}
ap.Print()
aps = append(aps, &ap)
if i+AuxPlatformFrmSize > len(data) {

8
extract/process.go
View File

@@ -6,6 +6,7 @@ import (
type Extractor struct {
params *Params
Clean bool
}
func NewExtractor(params *Params) *Extractor {
@@ -13,3 +14,10 @@ func NewExtractor(params *Params) *Extractor {
os.MkdirAll(params.TempPath, 0755)
return &Extractor{params: params}
}
func (e *Extractor) Cleanup() error {
if e.Clean {
os.RemoveAll(e.params.TempPath)
}
return nil
}

5
extract/trans_frame.go
View File

@@ -83,6 +83,11 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) {
snRange[fileno] = append(snRange[fileno], tf.SNo)
fsn.WriteString(fmt.Sprintf("%d %d %d\n", i, tf.SNo, fileno))
if tf.SecretFlag == 0xAA {
log.Info("secret frame is not supported")
break
}
// 只保留非空帧
if tf.FrameFlag != 0x55 && fileno != 0 {
datSet[fileno] = append(datSet[fileno], tf.Data...)