lankun/sjy01-preprocessing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor

Browse Source
This commit is contained in:
nuknal
2024-05-17 23:53:09 +08:00
parent 0d59c8514b
commit 5b3c4aa0a4
3 changed files with 196 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

270
extract/aux.go
View File

@@ -1,14 +1,12 @@
package extract
import (
"bufio"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"os"
"path/filepath"
"sort"
"strings"
"time"
@@ -32,8 +30,8 @@ type AuxFrameHead struct {
// B0 全色状态 0：不输出1：输出
ImgMode byte
SerialNo uint32 // 当前流水号
TimeSec uint32 // 时间（秒） 量纲：1us/bit，十六进制无符号整型数
TimeSecFrac uint32 // 秒小数
TimeSec uint32 // 时间（秒）
TimeSecFrac uint32 // 秒小数 量纲：1us/bit，十六进制无符号整型数
FileNo uint8 // 文件号
IsValidFrmHead bool
@@ -107,130 +105,36 @@ func (afh AuxFrameHead) CheckFrmHead() bool {
return false
}
type Record struct {
index int
frmHead *AuxFrameHead
}
// 从传输帧文件中分离辅助数据，分别存储到辅助数据文件 _AUX.dat 和图像数据文件 _IMG_{波谱}.dat 中
func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) error {
func (e *Extractor) SeprateAuxAndImgData(dataFile string, segmentIndex int) error {
// 打开传输帧文件 - 一次读入内存
sData, err := os.ReadFile(sDataFile)
data, err := os.ReadFile(dataFile)
if err != nil {
log.Println("read data from", sDataFile, "error:", err.Error())
log.Println("read data from", dataFile, "error:", err.Error())
return err
}
log.Info("seprate aux and img data from", sDataFile)
log.Info("seprate aux and img data from", dataFile)
// 数据质量分析
fimg, _ := os.Create("demo/temp/" + p.params.DataId + "_aux_img.txt")
defer fimg.Close()
fimg.WriteString("字节数 帧头流水号 文件号 帧头时间 中心辅助数据前4字节(行33-36)\n")
var qmap []*Record
var startAuxLine bool
var preSN uint32
for i := 0; i < len(sData); {
// 解析帧
if i+24 > len(sData) {
log.Info("length of original image frame head is not engough: ", len(sData)-i)
break
}
if sData[i] == 0xD1 && sData[i+1] == 0x5B && sData[i+2] == 0xD1 && sData[i+3] == 0x5B {
log.Debug("find package head: 0xD15BD15B")
} else {
i++
// log.Println(i,"not find package head: 0xD15BD15B, skip 1 byte")
continue
}
afh := &AuxFrameHead{}
afh.Decode(sData[i : i+24])
if !afh.IsValidFrmHead {
log.Debugf("[%d] invalid frame head of original raw data %v", i, afh.FrmHead)
i += 1
continue
}
r := &Record{index: i, frmHead: afh}
qmap = append(qmap, r)
utcTime := binary.BigEndian.Uint32(sData[i+32 : i+36])
t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), int64(afh.TimeSecFrac)*1000)
tAux := time.Unix(int64(utcTime+uint32(ReferenceTime2000)), 0)
startAuxLine = tAux.Year() == t.Year() && tAux.Month() == t.Month() && tAux.Day() == t.Day() &&
tAux.Hour() == t.Hour() && tAux.Minute() == t.Minute()
if !startAuxLine {
// fimg.WriteString(fmt.Sprintf("-----%d not AUX Start -----\n", afh.SerialNo))
} else {
fimg.WriteString(fmt.Sprintf("%d %d %d %s %s\n",
i,
afh.SerialNo,
afh.FileNo,
t.String(),
tAux.String(),
))
if afh.SerialNo-preSN != 16 {
fmt.Println("serial number not continuous", afh.SerialNo, preSN)
}
preSN = afh.SerialNo
}
i += afh.RowLength
}
sort.Slice(qmap, func(i, j int) bool {
return qmap[i].index < qmap[j].index
})
name := filepath.Base(sDataFile)
name := filepath.Base(dataFile)
name = strings.TrimRight(name, filepath.Ext(name))
outputDir := p.params.OutputPath
outputDir := e.params.OutputPath
aux0 := outputDir + "/" + name + "_EB.AUX"
os.Remove(aux0)
faux0, _ := os.OpenFile(aux0, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
waux0 := bufio.NewWriter(faux0)
defer waux0.Flush()
defer faux0.Close()
lw := newL0Writer(outputDir, name)
defer lw.Close()
aux1 := outputDir + "/" + name + "_PLAT.AUX"
os.Remove(aux1)
faux1, _ := os.OpenFile(aux1, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
waux1 := bufio.NewWriter(faux1)
defer waux1.Flush()
defer faux1.Close()
pan := outputDir + "/" + name + "_PAN.RAW"
os.Remove(pan)
fpan, _ := os.OpenFile(pan, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
wpan := bufio.NewWriter(fpan)
defer wpan.Flush()
defer fpan.Close()
panEnviHdr := EnviHdr{}
wpanHdr, _ := NewBSQWriter(outputDir+"/"+name+"_PAN.HDR", &panEnviHdr)
defer wpanHdr.Close()
// 先按单波段存储，再按波段组合存储为 BSQ 格式的 MSS
mss := outputDir + "/" + name + "_MSS.RAW"
os.Remove(mss)
fmss, _ := os.OpenFile(mss, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
wmss := bufio.NewWriter(fmss)
defer wmss.Flush()
defer fmss.Close()
mssEnviHdr := EnviHdr{}
wmssHdr, _ := NewBSQWriter(outputDir+"/"+name+"_MSS.HDR", &mssEnviHdr)
defer wmssHdr.Close()
var panEnviHdr, mssEnviHdr EnviHdr
var afh AuxFrameHead
dataLen := len(sData)
dataLen := len(data)
log.Info("original data length:", dataLen)
data = e.trimImgRawData(data)
dataLen = len(data)
log.Info("trimmed data length:", dataLen)
mssData := make([][]byte, 4)
e.quanlityAnalysis(data)
msdata := make([][]byte, 4)
var firstLineFound bool
var totalLines int
var pancount int
@@ -241,7 +145,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
}
// 包头
if sData[i] == 0xD1 && sData[i+1] == 0x5B && sData[i+2] == 0xD1 && sData[i+3] == 0x5B {
if data[i] == 0xD1 && data[i+1] == 0x5B && data[i+2] == 0xD1 && data[i+3] == 0x5B {
log.Debug("find package head: 0xD15BD15B")
} else {
i++
@@ -254,7 +158,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
log.Info("length of frame head is not engough for frame head")
break
}
afh.Decode(sData[i : i+24])
afh.Decode(data[i : i+24])
// pp.Println(afh)
if !afh.IsValidFrmHead {
@@ -269,7 +173,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
break
}
// 读取一行数据 sData[i : i+afh.RowLength]
// 读取一行数据 data[i : i+afh.RowLength]
if i+afh.RowLength > dataLen {
log.Infof("length of image row data %v is not enough: %v", dataLen-i, afh.RowLength)
break
@@ -277,7 +181,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
// 通过中心辅助时间判断是否第一行数据
if !firstLineFound {
utcTime := binary.BigEndian.Uint32(sData[i+32 : i+36])
utcTime := binary.BigEndian.Uint32(data[i+32 : i+36])
t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), int64(afh.TimeSecFrac)*1000)
tAux := time.Unix(int64(utcTime+uint32(ReferenceTime2000)), 0)
firstLineFound = tAux.Year() == t.Year() && tAux.Month() == t.Month() && tAux.Day() == t.Day() &&
@@ -292,39 +196,39 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
// 存储辅助数据到临时文件
dataIndex := i + 24
waux0.Write(sData[dataIndex : dataIndex+8]) // 8字节焦面电箱辅助数据
lw.ws[EB_AUX].w.Write(data[dataIndex : dataIndex+8]) // 8字节焦面电箱辅助数据
dataIndex += 8
waux1.Write(sData[dataIndex : dataIndex+32]) // 32字节中心机辅助数据
lw.ws[PLAT_AUX].w.Write(data[dataIndex : dataIndex+32]) // 32字节中心机辅助数据
dataIndex += 32
// 存储图像数据到临时文件 - 以 ENVI BSQ 格式存储，同时提供 HDR 描述文件
if afh.B0 {
// wpan.Write(sData[dataIndex : dataIndex+19040])
write16bPixelLittleEndian(wpan, sData[dataIndex:dataIndex+19040])
// wpan.Write(data[dataIndex : dataIndex+19040])
write16bPixelLittleEndian(lw.ws[PAN_RAW].w, data[dataIndex:dataIndex+19040])
dataIndex += 19040
panEnviHdr.Lines += 1
pancount += 19040
}
if afh.B1 {
// write16bPixelLittleEndian(wmss, sData[dataIndex:dataIndex+1192])
mssData[0] = append(mssData[0], sData[dataIndex:dataIndex+1192]...)
// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
msdata[0] = append(msdata[0], data[dataIndex:dataIndex+1192]...)
dataIndex += 1192
mssEnviHdr.Lines += 1
}
if afh.B2 {
// write16bPixelLittleEndian(wmss, sData[dataIndex:dataIndex+1192])
mssData[1] = append(mssData[1], sData[dataIndex:dataIndex+1192]...)
// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
msdata[1] = append(msdata[1], data[dataIndex:dataIndex+1192]...)
dataIndex += 1192
}
if afh.B3 {
// write16bPixelLittleEndian(wmss, sData[dataIndex:dataIndex+1192])
mssData[2] = append(mssData[2], sData[dataIndex:dataIndex+1192]...)
// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
msdata[2] = append(msdata[2], data[dataIndex:dataIndex+1192]...)
dataIndex += 1192
}
if afh.B4 {
// write16bPixelLittleEndian(wmss, sData[dataIndex:dataIndex+1192])
mssData[3] = append(mssData[3], sData[dataIndex:dataIndex+1192]...)
// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
msdata[3] = append(msdata[3], data[dataIndex:dataIndex+1192]...)
dataIndex += 1192
}
i = dataIndex // 完成一行数据解析
@@ -332,9 +236,9 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
var bands = 0
for i := 0; i < 4; i++ {
if len(mssData[i]) > 0 {
if len(msdata[i]) > 0 {
log.Println("write mss data of band B", i+1)
// _, err := write16bPixelLittleEndian(wmss, mssData[i])
_, err := write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[i])
if err != nil {
log.Error("write mss data error:", err.Error())
}
@@ -342,24 +246,104 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
}
}
write16bPixelLittleEndian(wmss, mssData[2])
panEnviHdr.Samples = 9520
panEnviHdr.Bands = 1
wpanHdr.Write([]byte(panEnviHdr.String()))
lw.ws[PAN_HDR].w.Write([]byte(panEnviHdr.String()))
mssEnviHdr.Lines = mssEnviHdr.Lines / 4 // 多光谱波段分别在 4 行中传输
mssEnviHdr.Samples = 2384
mssEnviHdr.Bands = 1
wmssHdr.Write([]byte(mssEnviHdr.String()))
lw.ws[MSS_HDR].w.Write([]byte(mssEnviHdr.String()))
return nil
}
// 写入 16 位像素值，小端序
func write16bPixelLittleEndian(w io.Writer, data []byte) (int, error) {
for i := 0; i < len(data)-1; i += 2 {
data[i], data[i+1] = data[i+1], data[i]
func (e *Extractor) quanlityAnalysis(data []byte) {
// 数据质量分析
fimg, _ := os.Create("demo/temp/" + e.params.DataId + "_aux_img.txt")
defer fimg.Close()
fimg.WriteString("字节数 帧头流水号 文件号 帧头时间 中心辅助数据前4字节(行33-36)\n")
var startAuxLine bool
var preSN uint32
for i := 0; i < len(data); {
afh := &AuxFrameHead{}
afh.Decode(data[i : i+24])
if !afh.IsValidFrmHead {
log.Error("invalid frame head of original raw data", i, len(data), afh.SerialNo, afh.FileNo)
return
}
t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), int64(afh.TimeSecFrac)*1000)
utcTime := binary.BigEndian.Uint32(data[i+32 : i+36])
tAux := time.Unix(int64(utcTime+uint32(ReferenceTime2000)), 0)
startAuxLine = math.Abs(float64(utcTime)-float64(afh.TimeSec)) < 30
if startAuxLine {
fimg.WriteString(fmt.Sprintf("%d %d %d %s %s\n",
i,
afh.SerialNo,
afh.FileNo,
t.String(),
tAux.String(),
))
if afh.SerialNo-preSN != 16 && preSN != 0 {
log.Println("serial number not continuous", afh.SerialNo, preSN)
}
preSN = afh.SerialNo
}
i += afh.RowLength
}
return w.Write(data)
}
// 裁剪图像数据，只保留有效数据
func (e *Extractor) trimImgRawData(data []byte) []byte {
var start, end int
// 将中心辅助数据时间合理的帧作为第一帧
var startAuxLine bool
for i := 0; i < len(data); {
// 解析帧
if i+24 > len(data) {
log.Info("length of original image frame head is not engough: ", len(data)-i)
break
}
if data[i] == 0xD1 && data[i+1] == 0x5B && data[i+2] == 0xD1 && data[i+3] == 0x5B {
log.Debug("find package head: 0xD15BD15B")
} else {
i++
continue
}
afh := &AuxFrameHead{}
afh.Decode(data[i : i+24])
if !afh.IsValidFrmHead {
log.Debugf("invalid frame head of original raw data %v", i)
i += 1
continue
}
utcTime := binary.BigEndian.Uint32(data[i+32 : i+36]) // 相对于2000年的秒数
startAuxLine = math.Abs(float64(utcTime)-float64(afh.TimeSec)) < 30 // 时间差小于30秒认为是有效数据
if !startAuxLine {
i += afh.RowLength
} else {
start = i
nLen := (len(data) - i - 1)
nLen = nLen - nLen%(afh.RowLength*16)
end = i + nLen
fmt.Println("start:", start, "end:", end, "i:", i, "len(data):", len(data),
len(data)%(afh.RowLength*16), afh.RowLength,
)
break
}
}
return data[start:end]
}

17
extract/envi.go
View File

@@ -50,20 +50,3 @@ func (h *EnviHdr) String() string {
return buf.String()
}
func NewBSQWriter(filepath string, content *EnviHdr) (*BSQWriter, error) {
fio, err := os.Create(filepath)
if err != nil {
return nil, err
}
w := fio
return &BSQWriter{filepath, fio, w, content}, nil
}
func (w *BSQWriter) Write(data []byte) (int, error) {
return w.w.Write(data)
}
func (w *BSQWriter) Close() error {
return w.fio.Close()
}

69
extract/writer.go Normal file
View File

@@ -0,0 +1,69 @@
package extract
import (
"bufio"
"io"
"os"
"path/filepath"
"github.com/sirupsen/logrus"
)
const (
EB_AUX = "EB.AUX"
PLAT_AUX = "PLAT.AUX"
PAN_RAW = "PAN_RAW"
MSS_RAW = "MSS_RAW"
PAN_HDR = "PAN_HDR"
MSS_HDR = "MSS_HDR"
MSS_ALL_IN_ONE = "MSS_ALL_IN_ONE"
)
type l0w struct {
w *bufio.Writer
f *os.File
name string
}
type L0Writer struct {
mssStoredType string
ws map[string]*l0w
}
func newL0Writer(outputDir, name string) *L0Writer {
var err error
suffix := []string{EB_AUX, PLAT_AUX, PAN_RAW, MSS_RAW, PAN_HDR, MSS_HDR}
lw := &L0Writer{
ws: make(map[string]*l0w),
mssStoredType: MSS_ALL_IN_ONE,
}
for _, s := range suffix {
w := &l0w{}
w.name = filepath.Join(outputDir, name+"_"+s)
w.f, err = os.OpenFile(w.name, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
if err != nil {
logrus.Panic("create file failed: ", w.name, err)
}
w.w = bufio.NewWriter(w.f)
lw.ws[s] = w
}
return lw
}
func (lw *L0Writer) Close() {
for _, w := range lw.ws {
w.w.Flush()
w.f.Close()
}
}
// 写入 16 位像素值，小端序
func write16bPixelLittleEndian(w io.Writer, data []byte) (int, error) {
for i := 0; i < len(data)-1; i += 2 {
data[i], data[i+1] = data[i+1], data[i]
}
return w.Write(data)
}