Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: upload failure due to time expiration #554

Merged
merged 8 commits into from
Jul 15, 2024
96 changes: 51 additions & 45 deletions internal/file/upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -319,11 +319,10 @@ type AuthSignParts struct {
}

type UploadInfo struct {
PartNum int
Bitmap *Bitmap
DBInfo *model_struct.LocalUpload
Resp *third.InitiateMultipartUploadResp
//Signs *AuthSignParts
PartNum int
Bitmap *Bitmap
DBInfo *model_struct.LocalUpload
Resp *third.InitiateMultipartUploadResp
CreateTime time.Time
BatchSignNum int32
f *File
Expand Down Expand Up @@ -417,45 +416,52 @@ func (u *UploadInfo) GetPartSign(ctx context.Context, partNumber int32) (*url.UR
return u.buildRequest(index)
}

func (f *File) getUpload(ctx context.Context, req *third.InitiateMultipartUploadReq) (*UploadInfo, error) {
func (f *File) getLocalUploadInfo(ctx context.Context, req *third.InitiateMultipartUploadReq) (info *UploadInfo) {
partNum := f.getPartNum(req.Size, req.PartSize)
var bitmap *Bitmap
if f.database != nil {
dbUpload, err := f.database.GetUpload(ctx, req.Hash)
if err == nil {
bitmapBytes, err := base64.StdEncoding.DecodeString(dbUpload.UploadInfo)
if err != nil || len(bitmapBytes) == 0 || partNum <= 1 || dbUpload.ExpireTime-3600*1000 < time.Now().UnixMilli() {
if err := f.database.DeleteUpload(ctx, req.Hash); err != nil {
return nil, err
}
dbUpload = nil
}
if dbUpload == nil {
bitmap = NewBitmap(partNum)
} else {
bitmap = ParseBitmap(bitmapBytes, partNum)
}
tUpInfo := &third.UploadInfo{
PartSize: req.PartSize,
Sign: &third.AuthSignParts{},
}
if dbUpload != nil {
tUpInfo.UploadID = dbUpload.UploadID
tUpInfo.ExpireTime = dbUpload.ExpireTime
if partNum <= 1 {
return nil
}
dbUpload, err := f.database.GetUpload(ctx, req.Hash)
if err != nil {
return nil
}
defer func() {
if info == nil {
if err := f.database.DeleteUpload(ctx, req.Hash); err != nil {
log.ZError(ctx, "delete upload db", err, "partHash", req.Hash)
}
return &UploadInfo{
PartNum: partNum,
Bitmap: bitmap,
DBInfo: dbUpload,
Resp: &third.InitiateMultipartUploadResp{
Upload: tUpInfo,
},
BatchSignNum: req.MaxParts,
f: f,
}, nil
}
log.ZError(ctx, "get upload db", err, "pratsMd5", req.Hash)
}()
if dbUpload.UploadID == "" || dbUpload.ExpireTime-3600*1000 < time.Now().UnixMilli() {
return nil
}
bitmapBytes, err := base64.StdEncoding.DecodeString(dbUpload.UploadInfo)
if err != nil {
log.ZError(ctx, "decode upload info", err, "partHash", req.Hash)
return nil
}
return &UploadInfo{
PartNum: partNum,
Bitmap: ParseBitmap(bitmapBytes, partNum),
DBInfo: dbUpload,
Resp: &third.InitiateMultipartUploadResp{
Upload: &third.UploadInfo{
PartSize: req.PartSize,
Sign: &third.AuthSignParts{},
UploadID: dbUpload.UploadID,
ExpireTime: dbUpload.ExpireTime,
},
},
BatchSignNum: req.MaxParts,
f: f,
}
}

func (f *File) getUpload(ctx context.Context, req *third.InitiateMultipartUploadReq) (*UploadInfo, error) {
if info := f.getLocalUploadInfo(ctx, req); info != nil {
return info, nil
}
partNum := f.getPartNum(req.Size, req.PartSize)
resp, err := f.initiateMultipartUploadResp(ctx, req)
if err != nil {
return nil, err
Expand All @@ -465,23 +471,23 @@ func (f *File) getUpload(ctx context.Context, req *third.InitiateMultipartUpload
Resp: resp,
}, nil
}
bitmap = NewBitmap(partNum)
bitmap := NewBitmap(partNum)
var dbUpload *model_struct.LocalUpload
if f.database != nil {
if partNum > 1 {
dbUpload = &model_struct.LocalUpload{
PartHash: req.Hash,
UploadID: resp.Upload.UploadID,
UploadInfo: base64.StdEncoding.EncodeToString(bitmap.Serialize()),
ExpireTime: resp.Upload.ExpireTime,
CreateTime: time.Now().UnixMilli(),
}
if err := f.database.DeleteUpload(ctx, req.Hash); err != nil {
log.ZError(ctx, "delete upload db", err, "partHash", req.Hash)
}
if err := f.database.InsertUpload(ctx, dbUpload); err != nil {
log.ZError(ctx, "insert upload db", err, "pratsHash", req.Hash, "name", req.Name)
}
}
if req.MaxParts >= 0 && len(resp.Upload.Sign.Parts) != int(req.MaxParts) {
resp.Upload.Sign.Parts = nil
}
return &UploadInfo{
PartNum: partNum,
Bitmap: bitmap,
Expand Down
Loading