356 lines
10 KiB
Go
356 lines
10 KiB
Go
package extract
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// 从传输帧文件中分离辅助数据，分别存储到辅助数据文件 _AUX.dat 和图像数据文件 _IMG_{波谱}.dat 中
|
|
func (e *Extractor) SeprateAuxAndImgData(dataFile string, segmentIndex int) error {
|
|
// 打开传输帧文件 - 一次读入内存
|
|
data, err := os.ReadFile(dataFile)
|
|
if err != nil {
|
|
log.Println("read data from", dataFile, "error:", err.Error())
|
|
return err
|
|
}
|
|
|
|
log.Info("seprate aux and img data from", dataFile)
|
|
|
|
dataLen := len(data)
|
|
log.Info("length of original data: ", dataLen)
|
|
var firstFrmHead *AuxFrameHead
|
|
data, firstFrmHead = e.trimImgRawData(data)
|
|
dataLen = len(data)
|
|
log.Info("after trim,length of data: ", dataLen)
|
|
e.quanlityAnalysis(data)
|
|
|
|
outputDir := e.params.OutputPath
|
|
name := strings.Join([]string{
|
|
e.params.Satellite,
|
|
"PMS",
|
|
time.Unix(int64(firstFrmHead.TimeSec)+int64(ReferenceTime2000),
|
|
int64(firstFrmHead.TimeSecFrac)*1000).
|
|
Format("20060102_150405"),
|
|
e.params.DataId,
|
|
fmt.Sprintf("%03d", firstFrmHead.FileNo),
|
|
}, "_")
|
|
|
|
lw := newL0Writer(outputDir, name)
|
|
defer lw.Close()
|
|
|
|
var panEnviHdr, mssEnviHdr EnviHdr
|
|
|
|
var afh AuxFrameHead
|
|
|
|
pandata := make([]byte, 0)
|
|
msdata := make([][]byte, 4)
|
|
var header []byte
|
|
var ebAux []byte
|
|
var platAux []byte
|
|
|
|
for i := 0; i < dataLen; {
|
|
if i+4 > dataLen {
|
|
log.Println("end of data, dataLen:", dataLen, "i:", i)
|
|
break
|
|
}
|
|
|
|
// 包头
|
|
if data[i] == 0xD1 && data[i+1] == 0x5B && data[i+2] == 0xD1 && data[i+3] == 0x5B {
|
|
log.Debug("find package head: 0xD15BD15B")
|
|
} else {
|
|
i++
|
|
continue
|
|
}
|
|
|
|
// 解析帧
|
|
if i+24 > dataLen {
|
|
log.Info("length of frame head is not engough for frame head")
|
|
break
|
|
}
|
|
afh.Decode(data[i : i+24])
|
|
|
|
if !afh.IsValidFrmHead {
|
|
log.Info("invalid frame head of original raw data")
|
|
i += 1
|
|
continue
|
|
}
|
|
|
|
// 目前只支持线阵模式
|
|
if !afh.IsLinerMatrix {
|
|
log.Error("not liner matrix mode, only support liner matrix mode")
|
|
break
|
|
}
|
|
|
|
// 读取一行数据 data[i : i+afh.RowLength]
|
|
if i+afh.RowLength > dataLen {
|
|
log.Infof("length of image row data %v is not enough: %v", dataLen-i, afh.RowLength)
|
|
break
|
|
}
|
|
|
|
auxTime := binary.BigEndian.Uint32(data[i+32 : i+36])
|
|
if math.Abs(float64(auxTime)-float64(afh.TimeSec)) < 30 {
|
|
header = append(header, data[i:i+24]...)
|
|
}
|
|
// 存储辅助数据到临时文件
|
|
dataIndex := i + 24
|
|
// lw.ws[EB_AUX].w.Write(data[dataIndex : dataIndex+8]) // 8字节焦面电箱辅助数据
|
|
ebAux = append(ebAux, data[dataIndex:dataIndex+8]...)
|
|
dataIndex += 8
|
|
// lw.ws[PLAT_AUX].w.Write(data[dataIndex : dataIndex+32]) // 32字节中心机辅助数据
|
|
platAux = append(platAux, data[dataIndex:dataIndex+32]...)
|
|
dataIndex += 32
|
|
|
|
if !afh.B0 || !afh.B1 || !afh.B2 || !afh.B3 || !afh.B4 {
|
|
log.Error("not all bands are available")
|
|
break
|
|
}
|
|
|
|
// 存储图像数据到临时文件 - 以 ENVI BSQ 格式存储，同时提供 HDR 描述文件
|
|
if afh.B0 {
|
|
// wpan.Write(data[dataIndex : dataIndex+19040])
|
|
pandata = append(pandata, data[dataIndex:dataIndex+19040]...)
|
|
dataIndex += 19040
|
|
}
|
|
|
|
if afh.B1 {
|
|
// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
|
|
msdata[0] = append(msdata[0], data[dataIndex:dataIndex+1192]...)
|
|
dataIndex += 1192
|
|
}
|
|
if afh.B2 {
|
|
// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
|
|
msdata[1] = append(msdata[1], data[dataIndex:dataIndex+1192]...)
|
|
dataIndex += 1192
|
|
}
|
|
if afh.B3 {
|
|
// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
|
|
msdata[2] = append(msdata[2], data[dataIndex:dataIndex+1192]...)
|
|
dataIndex += 1192
|
|
}
|
|
if afh.B4 {
|
|
// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
|
|
msdata[3] = append(msdata[3], data[dataIndex:dataIndex+1192]...)
|
|
dataIndex += 1192
|
|
}
|
|
i = dataIndex // 完成一行数据解析
|
|
}
|
|
|
|
if len(msdata[0]) != len(msdata[1]) || len(msdata[2]) != len(msdata[3]) {
|
|
log.Error("mss data of bands B1-B4 are not equal")
|
|
return errors.New("mss data of bands B1-B4 are not equal")
|
|
}
|
|
|
|
// 先判断辅助数据是否完整
|
|
// 帧头+辅助数据
|
|
if len(header)/24 < len(ebAux)/128 || len(ebAux)/128 != len(platAux)/512 {
|
|
fmt.Println("aux data length:", len(header)/24, len(ebAux)/128, len(platAux)/512)
|
|
return errors.New("aux data length is not equal")
|
|
}
|
|
|
|
auxRows := len(header) / 24
|
|
for i := 0; i < auxRows; i++ {
|
|
lw.ws[AUX].w.Write(header[i*24 : (i+1)*24])
|
|
lw.ws[AUX].w.Write(ebAux[i*128 : (i+1)*128])
|
|
lw.ws[AUX].w.Write(platAux[i*512 : (i+1)*512])
|
|
}
|
|
|
|
_, _, aps, err := e.ExtractAux(lw.ws[AUX].name, lw.ws[AUX].name+".xlsx")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
auxHeight := len(aps) * 16 // 16行图像数据为一组辅助数据
|
|
panHeight := len(pandata) / 19040
|
|
mssHeight := 4 * len(msdata[0]) / 4768
|
|
log.Println("pan height:", panHeight, "mss height:", mssHeight, "aux height:", auxHeight)
|
|
// 取最小值作为有效数据长度
|
|
efficientHeight := int(math.Min(float64(panHeight), float64(mssHeight)))
|
|
efficientHeight = int(math.Min(float64(efficientHeight), float64(auxHeight)))
|
|
log.Println("efficient height:", efficientHeight)
|
|
|
|
// 写入pan数据
|
|
for i := 0; i < len(pandata); i += 19040 {
|
|
start := i + (2+48+38)*2
|
|
end := start + 18688
|
|
write16bPixelLittleEndian(lw.ws[PAN_RAW].w, pandata[start:end])
|
|
panEnviHdr.Lines += 1
|
|
if panEnviHdr.Lines >= efficientHeight {
|
|
break
|
|
}
|
|
}
|
|
|
|
mssRowLen := 1192 * 4
|
|
for i := 0; i < len(msdata[0]); i += mssRowLen {
|
|
var err error
|
|
start := i + (1+11+10)*2
|
|
end := start + 4672
|
|
_, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[0][start:end])
|
|
if err != nil {
|
|
log.Error("write mss 1 data error:", err.Error())
|
|
}
|
|
_, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[1][start:end])
|
|
if err != nil {
|
|
log.Error("write mss 2 data error:", err.Error())
|
|
}
|
|
_, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[2][start:end])
|
|
if err != nil {
|
|
log.Error("write mss 3 data error:", err.Error())
|
|
}
|
|
_, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[3][start:end])
|
|
if err != nil {
|
|
log.Error("write mss 4 data error:", err.Error())
|
|
}
|
|
|
|
mssEnviHdr.Lines += 1
|
|
if mssEnviHdr.Lines*4 >= efficientHeight {
|
|
break
|
|
}
|
|
}
|
|
|
|
panEnviHdr.Samples = 9344
|
|
panEnviHdr.Bands = 1
|
|
lw.ws[PAN_HDR].w.Write([]byte(panEnviHdr.String()))
|
|
|
|
mssEnviHdr.Samples = 2336 * 4
|
|
mssEnviHdr.Bands = 1
|
|
lw.ws[MSS_HDR].w.Write([]byte(mssEnviHdr.String()))
|
|
|
|
fmt.Println("pan height:", panEnviHdr.Lines, "mss height:", mssEnviHdr.Lines)
|
|
|
|
e.mutex.Lock()
|
|
defer e.mutex.Unlock()
|
|
seg := Segment{
|
|
Pan: lw.ws[PAN_RAW].name,
|
|
Mss: lw.ws[MSS_RAW].name,
|
|
Aux: lw.ws[AUX].name,
|
|
Id: strings.Split(filepath.Base(lw.ws[AUX].name), ".")[0],
|
|
StartTime: XMLTime{
|
|
TimeZone: "UTC",
|
|
Value: time.Unix(int64(aps[0].UTCTimeSec)+int64(ReferenceTime2000), int64(aps[0].Microsecond)*1000).UTC().Format(time.RFC3339),
|
|
},
|
|
EndTime: XMLTime{
|
|
TimeZone: "UTC",
|
|
Value: time.Unix(int64(aps[len(aps)-1].UTCTimeSec)+int64(ReferenceTime2000), int64(aps[len(aps)-1].Microsecond)*1000).UTC().Format(time.RFC3339),
|
|
},
|
|
}
|
|
e.report.Segments = append(e.report.Segments, seg)
|
|
|
|
return nil
|
|
}
|
|
|
|
func (e *Extractor) quanlityAnalysis(data []byte) {
|
|
// 数据质量分析
|
|
fimg, _ := os.Create("demo/temp/" + e.params.DataId + "/aux.txt")
|
|
defer fimg.Close()
|
|
fimg.WriteString("字节数 帧头流水号 文件号 帧头时间 中心辅助数据前4字节(行33-36)\n")
|
|
|
|
var startAuxLine bool
|
|
var preSN uint32
|
|
for i := 0; i < len(data); {
|
|
afh := &AuxFrameHead{}
|
|
afh.Decode(data[i : i+24])
|
|
|
|
if !afh.IsValidFrmHead {
|
|
log.Errorf("invalid frame head of original raw data i: %v, len: %v", i, len(data))
|
|
return
|
|
}
|
|
|
|
t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), int64(afh.TimeSecFrac)*1000)
|
|
|
|
utcTime := binary.BigEndian.Uint32(data[i+32 : i+36])
|
|
tAux := time.Unix(int64(utcTime+uint32(ReferenceTime2000)), 0)
|
|
|
|
startAuxLine = math.Abs(float64(utcTime)-float64(afh.TimeSec)) < 30
|
|
if startAuxLine {
|
|
fimg.WriteString(fmt.Sprintf("%d %d %d %s %s\n",
|
|
i,
|
|
afh.SerialNo,
|
|
afh.FileNo,
|
|
t.String(),
|
|
tAux.String(),
|
|
))
|
|
|
|
if afh.SerialNo-preSN != 16 && preSN != 0 {
|
|
log.Println("serial number not continuous", afh.SerialNo, preSN)
|
|
}
|
|
preSN = afh.SerialNo
|
|
}
|
|
|
|
i += afh.RowLength
|
|
}
|
|
}
|
|
|
|
// 裁剪图像数据，只保留有效数据
|
|
func (e *Extractor) trimImgRawData(data []byte) ([]byte, *AuxFrameHead) {
|
|
var start, end int
|
|
afh := &AuxFrameHead{}
|
|
// 将中心辅助数据时间合理的帧作为第一帧
|
|
var startAuxLine bool
|
|
for i := 0; i < len(data); {
|
|
// 解析帧
|
|
if i+24 > len(data) {
|
|
log.Info("length of original image frame head is not engough: ", len(data)-i)
|
|
break
|
|
}
|
|
|
|
if data[i] == 0xD1 && data[i+1] == 0x5B && data[i+2] == 0xD1 && data[i+3] == 0x5B {
|
|
log.Debug("find package head: 0xD15BD15B")
|
|
} else {
|
|
i++
|
|
continue
|
|
}
|
|
|
|
afh.Decode(data[i : i+24])
|
|
|
|
if !afh.IsValidFrmHead {
|
|
log.Debugf("invalid frame head of original raw data %v", i)
|
|
i += 1
|
|
continue
|
|
}
|
|
|
|
utcTime := binary.BigEndian.Uint32(data[i+32 : i+36]) // 相对于2000年的秒数
|
|
startAuxLine = math.Abs(float64(utcTime)-float64(afh.TimeSec)) < 30 // 时间差小于30秒认为是有效数据
|
|
if !startAuxLine {
|
|
i += afh.RowLength
|
|
} else {
|
|
start = i
|
|
nLen := (len(data) - i - 1)
|
|
nLen = nLen - nLen%(afh.RowLength*16)
|
|
end = i + nLen
|
|
break
|
|
}
|
|
}
|
|
|
|
return data[start:end], afh
|
|
}
|
|
|
|
// 9520 16bit 像素 19040 字节，返回有效像素字节起始位置
|
|
// 9520 像素 2(Margin) + 48(OB) + 38(Margin) + 9344(Effective Pixels) + 38(Margin) + 48(OB) + 2(Margin)
|
|
// 7056 像素 2(Margin) + 12(OB) + 18(Margin) + 20(Margin) + 7000(Effective Pixels) + 4(Margin)
|
|
func PanEffectivePixel(row []byte) (start, end int) {
|
|
if len(row) < 19040 {
|
|
return -1, -1
|
|
}
|
|
start = (0 + 2 + 48 + 38) * 2
|
|
end = 19040 - (2+48+28)*2
|
|
return start, end
|
|
}
|
|
|
|
// 2384 像素 1+11+10+2336+10+8+8
|
|
func MssEffectivePixel(row []byte) (start, end int) {
|
|
if len(row) < 4768 {
|
|
return -1, -1
|
|
}
|
|
start = (1 + 11 + 10) * 2
|
|
end = 4768 - (10+8+8)*2
|
|
return start, end
|
|
}
|