diff --git a/go.mod b/go.mod index 517bcdc1..6d37a691 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,16 @@ go 1.23 require ( github.com/MicahParks/keyfunc/v2 v2.1.0 + github.com/aws/aws-sdk-go-v2 v1.40.0 + github.com/aws/aws-sdk-go-v2/config v1.32.2 + github.com/aws/aws-sdk-go-v2/credentials v1.19.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1 github.com/bytedance/sonic v1.12.1 github.com/go-playground/validator/v10 v10.27.0 github.com/gofiber/contrib/jwt v1.0.10 github.com/gofiber/fiber/v2 v2.52.5 github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/google/uuid v1.6.0 github.com/jackc/pgconn v1.14.1 github.com/redis/go-redis/v9 v9.14.0 github.com/sirupsen/logrus v1.9.3 @@ -20,6 +25,21 @@ require ( require ( github.com/andybalholm/brotli v1.1.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 // indirect + github.com/aws/smithy-go v1.23.2 // indirect github.com/bytedance/sonic/loader v0.2.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect @@ -30,7 +50,6 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgio v1.0.0 // indirect diff --git a/go.sum b/go.sum index c07e37e3..71f9378c 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,44 @@ github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+G github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc= +github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y= +github.com/aws/aws-sdk-go-v2/config v1.32.2 h1:4liUsdEpUUPZs5WVapsJLx5NPmQhQdez7nYFcovrytk= +github.com/aws/aws-sdk-go-v2/config v1.32.2/go.mod h1:l0hs06IFz1eCT+jTacU/qZtC33nvcnLADAPL/XyrkZI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.2 h1:qZry8VUyTK4VIo5aEdUcBjPZHL2v4FyQ3QEOaWcFLu4= +github.com/aws/aws-sdk-go-v2/credentials v1.19.2/go.mod h1:YUqm5a1/kBnoK+/NY5WEiMocZihKSo15/tJdmdXnM5g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14/go.mod h1:1ipeGBMAxZ0xcTm6y6paC2C/J6f6OO7LBODV9afuAyM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14/go.mod h1:k1xtME53H1b6YpZt74YmwlONMWf4ecM+lut1WQLAF/U= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 h1:Hjkh7kE6D81PgrHlE/m9gx+4TyyeLHuY8xJs7yXN5C4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5/go.mod h1:nPRXgyCfAurhyaTMoBMwRBYBhaHI4lNPAnJmjM0Tslc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE46kyYqyhs0XEBDFFSREtdnr8HQuLPQPLCrY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14/go.mod h1:s1ydyWG9pm3ZwmmYN21HKyG9WzAZhYVW85wMHs5FV6w= +github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1 h1:OgQy/+0+Kc3khtqiEOk23xQAglXi3Tj0y5doOxbi5tg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 h1:MxMBdKTYBjPQChlJhi4qlEueqB1p1KcbTEa7tD5aqPs= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.2/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 h1:ksUT5KtgpZd3SAiFJNJ0AFEJVva3gjBmN7eXUZjzUwQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.5/go.mod h1:av+ArJpoYf3pgyrj6tcehSFW+y9/QvAY8kMooR9bZCw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 h1:GtsxyiF3Nd3JahRBJbxLCCdYW9ltGQYrFWg8XdkGDd8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 h1:a5UTtD4mHBU3t0o6aHQZFJTNKVfxFWfPX7J0Lr7G+uY= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.2/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso= +github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM= +github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= diff --git a/internal/common/repository/common.document.repository.go b/internal/common/repository/common.document.repository.go new file mode 100644 index 00000000..79e8a04d --- /dev/null +++ b/internal/common/repository/common.document.repository.go @@ -0,0 +1,62 @@ +package repository + +import ( + "context" + + entity "gitlab.com/mbugroup/lti-api.git/internal/entities" + "gorm.io/gorm" +) + +type DocumentRepository interface { + BaseRepository[entity.Document] + ListByTarget(ctx context.Context, documentableType string, documentableID uint64, modifier func(*gorm.DB) *gorm.DB) ([]entity.Document, error) + DeleteByTarget(ctx context.Context, documentableType string, documentableID uint64, modifier func(*gorm.DB) *gorm.DB) error +} + +type documentRepositoryImpl struct { + *BaseRepositoryImpl[entity.Document] +} + +func NewDocumentRepository(db *gorm.DB) DocumentRepository { + return &documentRepositoryImpl{ + BaseRepositoryImpl: NewBaseRepository[entity.Document](db), + } +} + +func (r *documentRepositoryImpl) ListByTarget( + ctx context.Context, + documentableType string, + documentableID uint64, + modifier func(*gorm.DB) *gorm.DB, +) ([]entity.Document, error) { + var documents []entity.Document + + q := r.DB().WithContext(ctx). + Where("documentable_type = ? AND documentable_id = ?", documentableType, documentableID) + + if modifier != nil { + q = modifier(q) + } + + if err := q.Order("created_at ASC").Find(&documents).Error; err != nil { + return nil, err + } + + return documents, nil +} + +func (r *documentRepositoryImpl) DeleteByTarget( + ctx context.Context, + documentableType string, + documentableID uint64, + modifier func(*gorm.DB) *gorm.DB, +) error { + q := r.DB().WithContext(ctx). + Where("documentable_type = ? AND documentable_id = ?", documentableType, documentableID) + + if modifier != nil { + q = modifier(q) + } + + return q.Delete(&entity.Document{}).Error +} diff --git a/internal/common/service/common.document.service.go b/internal/common/service/common.document.service.go new file mode 100644 index 00000000..fe2a41cc --- /dev/null +++ b/internal/common/service/common.document.service.go @@ -0,0 +1,411 @@ +package service + +import ( + "context" + "errors" + "fmt" + "mime" + "mime/multipart" + "path/filepath" + "strings" + + commonRepo "gitlab.com/mbugroup/lti-api.git/internal/common/repository" + "gitlab.com/mbugroup/lti-api.git/internal/config" + entity "gitlab.com/mbugroup/lti-api.git/internal/entities" + "gitlab.com/mbugroup/lti-api.git/internal/utils" + + "github.com/google/uuid" +) + +const ( + defaultDocumentPathLimit = 50 + defaultDocumentKeyPrefix = "docs" + maxDocumentNameLength = 50 +) + +type DocumentService interface { + UploadDocuments(ctx context.Context, req DocumentUploadRequest) ([]DocumentUploadResult, error) + ListByTarget(ctx context.Context, documentableType string, documentableID uint64) ([]entity.Document, error) + DeleteDocuments(ctx context.Context, ids []uint, removeFromStorage bool) error + DeleteByTarget(ctx context.Context, documentableType string, documentableID uint64, removeFromStorage bool) error + PublicURL(document entity.Document) string +} + +type DocumentUploadRequest struct { + DocumentableType string + DocumentableID uint64 + CreatedBy *uint + Files []DocumentFile +} + +type DocumentFile struct { + File *multipart.FileHeader + Type string + Index *int +} + +type DocumentUploadResult struct { + Document entity.Document + URL string + Index *int +} + +type DocumentServiceOption func(*documentService) + +type documentService struct { + repo commonRepo.DocumentRepository + storage DocumentStorage + keyPrefix string + maxPathLength int +} + +func NewDocumentService(repo commonRepo.DocumentRepository, storage DocumentStorage, opts ...DocumentServiceOption) DocumentService { + svc := &documentService{ + repo: repo, + storage: storage, + keyPrefix: defaultDocumentKeyPrefix, + maxPathLength: defaultDocumentPathLimit, + } + + for _, opt := range opts { + opt(svc) + } + + return svc +} + +func NewDocumentServiceFromConfig(ctx context.Context, repo commonRepo.DocumentRepository) (DocumentService, error) { + if repo == nil { + return nil, errors.New("document repository is required") + } + if strings.TrimSpace(config.S3Bucket) == "" { + return nil, errors.New("S3_BUCKET is not configured") + } + + storage, err := NewS3DocumentStorage(ctx, S3DocumentStorageConfig{ + Region: config.S3Region, + Bucket: config.S3Bucket, + AccessKey: config.S3AccessKey, + SecretKey: config.S3SecretKey, + Endpoint: config.S3Endpoint, + BaseURL: config.S3PublicBaseURL, + ForcePathStyle: config.S3ForcePathStyle, + }) + if err != nil { + return nil, err + } + + prefix := config.S3DocumentKeyPrefix + if prefix == "" { + prefix = defaultDocumentKeyPrefix + } + + return NewDocumentService( + repo, + storage, + WithDocumentKeyPrefix(prefix), + WithDocumentPathLimit(defaultDocumentPathLimit), + ), nil +} + +func WithDocumentKeyPrefix(prefix string) DocumentServiceOption { + return func(svc *documentService) { + prefix = strings.Trim(prefix, "/") + if prefix == "" { + prefix = defaultDocumentKeyPrefix + } + svc.keyPrefix = prefix + } +} + +func WithDocumentPathLimit(limit int) DocumentServiceOption { + return func(svc *documentService) { + if limit > 0 { + svc.maxPathLength = limit + } + } +} + +func (s *documentService) UploadDocuments(ctx context.Context, req DocumentUploadRequest) ([]DocumentUploadResult, error) { + if s.repo == nil { + return nil, errors.New("document repository not configured") + } + if s.storage == nil { + return nil, errors.New("document storage not configured") + } + + documentableType := strings.ToUpper(strings.TrimSpace(req.DocumentableType)) + if documentableType == "" { + return nil, errors.New("documentable type is required") + } + if req.DocumentableID == 0 { + return nil, errors.New("documentable id is required") + } + if len(req.Files) == 0 { + return nil, errors.New("no files to upload") + } + + var createdBy *uint + if req.CreatedBy != nil && *req.CreatedBy != 0 { + idCopy := *req.CreatedBy + createdBy = &idCopy + } + + results := make([]DocumentUploadResult, 0, len(req.Files)) + createdDocs := make([]entity.Document, 0, len(req.Files)) + + for _, file := range req.Files { + if file.File == nil { + return nil, errors.New("file header is required") + } + + originalName := sanitizeDocumentName(file.File.Filename) + contentType := detectContentType(file.File, originalName) + ext := detectExtension(file.File.Filename, contentType) + key, err := s.generateObjectKey(ext) + if err != nil { + s.rollbackDocuments(ctx, createdDocs) + return nil, err + } + + reader, err := file.File.Open() + if err != nil { + s.rollbackDocuments(ctx, createdDocs) + return nil, err + } + uploadRes, err := s.storage.Upload(ctx, key, reader, file.File.Size, contentType) + _ = reader.Close() + if err != nil { + s.rollbackDocuments(ctx, createdDocs) + return nil, err + } + + docType := resolveDocumentType(file.Type, documentableType) + doc := entity.Document{ + DocumentableType: documentableType, + DocumentableId: req.DocumentableID, + Type: docType, + Path: uploadRes.Key, + Name: originalName, + Ext: strings.TrimPrefix(ext, "."), + Size: float64(file.File.Size), + CreatedBy: createdBy, + } + + if err := s.repo.CreateOne(ctx, &doc, nil); err != nil { + _ = s.storage.Delete(ctx, uploadRes.Key) + s.rollbackDocuments(ctx, createdDocs) + return nil, err + } + + createdDocs = append(createdDocs, doc) + results = append(results, DocumentUploadResult{ + Document: doc, + URL: uploadRes.URL, + Index: cloneIndex(file.Index), + }) + } + + return results, nil +} + +func (s *documentService) ListByTarget(ctx context.Context, documentableType string, documentableID uint64) ([]entity.Document, error) { + if s.repo == nil { + return nil, errors.New("document repository not configured") + } + + documentableType = strings.ToUpper(strings.TrimSpace(documentableType)) + if documentableType == "" { + return nil, errors.New("documentable type is required") + } + if documentableID == 0 { + return nil, errors.New("documentable id is required") + } + + return s.repo.ListByTarget(ctx, documentableType, documentableID, nil) +} + +func (s *documentService) DeleteDocuments(ctx context.Context, ids []uint, removeFromStorage bool) error { + if s.repo == nil { + return errors.New("document repository not configured") + } + if len(ids) == 0 { + return nil + } + + docs, err := s.repo.GetByIDs(ctx, ids, nil) + if err != nil { + return err + } + + for _, doc := range docs { + if err := s.repo.DeleteOne(ctx, doc.Id); err != nil { + return err + } + if removeFromStorage && s.storage != nil { + if err := s.storage.Delete(ctx, doc.Path); err != nil { + utils.Log.WithError(err).Warnf("failed to delete document object %s", doc.Path) + } + } + } + + return nil +} + +func (s *documentService) DeleteByTarget(ctx context.Context, documentableType string, documentableID uint64, removeFromStorage bool) error { + if s.repo == nil { + return errors.New("document repository not configured") + } + + documentableType = strings.ToUpper(strings.TrimSpace(documentableType)) + if documentableType == "" || documentableID == 0 { + return errors.New("documentable type and id are required") + } + + var docs []entity.Document + if removeFromStorage && s.storage != nil { + var err error + docs, err = s.repo.ListByTarget(ctx, documentableType, documentableID, nil) + if err != nil { + return err + } + } + + if err := s.repo.DeleteByTarget(ctx, documentableType, documentableID, nil); err != nil { + return err + } + + if removeFromStorage && len(docs) > 0 { + for _, doc := range docs { + if err := s.storage.Delete(ctx, doc.Path); err != nil { + utils.Log.WithError(err).Warnf("failed to delete document object %s", doc.Path) + } + } + } + + return nil +} + +func (s *documentService) PublicURL(document entity.Document) string { + if s.storage == nil || strings.TrimSpace(document.Path) == "" { + return "" + } + return s.storage.URL(document.Path) +} + +func (s *documentService) generateObjectKey(ext string) (string, error) { + normalizedExt := strings.TrimSpace(ext) + if normalizedExt != "" && !strings.HasPrefix(normalizedExt, ".") { + normalizedExt = "." + normalizedExt + } + + u := uuid.New().String() + key := fmt.Sprintf("%s/%s%s", strings.Trim(s.keyPrefix, "/"), u, normalizedExt) + if s.keyPrefix == "" { + key = fmt.Sprintf("%s%s", u, normalizedExt) + } + + if len(key) > s.maxPathLength { + key = fmt.Sprintf("%s%s", u, normalizedExt) + } + + if len(key) > s.maxPathLength { + return "", fmt.Errorf("object key exceeds maximum length (%d)", s.maxPathLength) + } + + return key, nil +} + +func (s *documentService) rollbackDocuments(ctx context.Context, docs []entity.Document) { + if len(docs) == 0 { + return + } + + for i := len(docs) - 1; i >= 0; i-- { + doc := docs[i] + if s.repo != nil && doc.Id != 0 { + if err := s.repo.DeleteOne(ctx, doc.Id); err != nil { + utils.Log.WithError(err).Warnf("failed to rollback document #%d", doc.Id) + } + } + if s.storage != nil && strings.TrimSpace(doc.Path) != "" { + if err := s.storage.Delete(ctx, doc.Path); err != nil { + utils.Log.WithError(err).Warnf("failed to rollback document object %s", doc.Path) + } + } + } +} + +func sanitizeDocumentName(name string) string { + name = filepath.Base(strings.TrimSpace(name)) + if name == "." || name == "" { + name = "document" + } + name = strings.Map(func(r rune) rune { + if r < 32 { + return -1 + } + switch r { + case '\\', '/', ':', '*', '?', '"', '<', '>', '|': + return '-' + default: + return r + } + }, name) + + if len(name) > maxDocumentNameLength { + runes := []rune(name) + if len(runes) > maxDocumentNameLength { + name = string(runes[:maxDocumentNameLength]) + } + } + return name +} + +func detectExtension(filename, contentType string) string { + ext := strings.ToLower(strings.TrimSpace(filepath.Ext(filename))) + if ext == "" && contentType != "" { + if exts, _ := mime.ExtensionsByType(contentType); len(exts) > 0 { + ext = exts[0] + } + } + if ext == "" { + return ".bin" + } + if !strings.HasPrefix(ext, ".") { + ext = "." + ext + } + return ext +} + +func detectContentType(file *multipart.FileHeader, filename string) string { + if file == nil { + return "application/octet-stream" + } + contentType := strings.TrimSpace(file.Header.Get("Content-Type")) + if contentType != "" { + return contentType + } + if ext := filepath.Ext(filename); ext != "" { + if guess := mime.TypeByExtension(ext); guess != "" { + return guess + } + } + return "application/octet-stream" +} + +func resolveDocumentType(fileType, fallback string) string { + value := strings.ToUpper(strings.TrimSpace(fileType)) + if value == "" { + return fallback + } + return value +} + +func cloneIndex(index *int) *int { + if index == nil { + return nil + } + value := *index + return &value +} diff --git a/internal/common/service/common.document.service_test.go b/internal/common/service/common.document.service_test.go new file mode 100644 index 00000000..8b7d248d --- /dev/null +++ b/internal/common/service/common.document.service_test.go @@ -0,0 +1,101 @@ +package service + +import ( + "bytes" + "context" + "mime/multipart" + "net/http/httptest" + "strings" + "testing" + + commonRepo "gitlab.com/mbugroup/lti-api.git/internal/common/repository" + "gitlab.com/mbugroup/lti-api.git/internal/config" + "gitlab.com/mbugroup/lti-api.git/internal/database" + entity "gitlab.com/mbugroup/lti-api.git/internal/entities" + "gorm.io/gorm" +) + +func TestDocumentServiceUpload(t *testing.T) { + if strings.TrimSpace(config.S3Bucket) == "" { + t.Fatal("S3 bucket is not configured; set S3_* env vars to run this test") + } + + ctx := context.Background() + db := setupDocumentTestDB(t) + repo := commonRepo.NewDocumentRepository(db) + + svc, err := NewDocumentServiceFromConfig(ctx, repo) + if err != nil { + t.Fatalf("failed to create document service from config: %v", err) + } + + file := newTestFileHeader(t, "integration-proof.txt", "text/plain", []byte("document integration test")) + userID := uint(100) + + results, err := svc.UploadDocuments(ctx, DocumentUploadRequest{ + DocumentableType: "INVENTORY_TRANSFER", + DocumentableID: 99, + CreatedBy: &userID, + Files: []DocumentFile{ + {File: file, Type: "integration"}, + }, + }) + if err != nil { + t.Fatalf("upload to S3 failed: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 uploaded document, got %d", len(results)) + } + + doc := results[0].Document + if doc.Path == "" { + t.Fatalf("expected non-empty storage path") + } + if results[0].URL == "" { + t.Fatalf("expected public URL for uploaded document") + } + + t.Logf("uploaded document #%d to %s (path=%s)", doc.Id, results[0].URL, doc.Path) +} + +func setupDocumentTestDB(t *testing.T) *gorm.DB { + t.Helper() + if strings.TrimSpace(config.DBHost) == "" || strings.TrimSpace(config.DBName) == "" { + t.Fatal("database configuration missing; ensure DB_HOST and DB_NAME are set") + } + db := database.Connect(config.DBHost, config.DBName) + if db == nil { + t.Fatal("failed to create database connection") + } + if err := db.AutoMigrate(&entity.Document{}); err != nil { + t.Fatalf("failed to migrate document table: %v", err) + } + return db +} + +func newTestFileHeader(t *testing.T, filename, contentType string, data []byte) *multipart.FileHeader { + t.Helper() + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("documents", filename) + if err != nil { + t.Fatalf("failed to create form file: %v", err) + } + if _, err := part.Write(data); err != nil { + t.Fatalf("failed to write file data: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("failed to close writer: %v", err) + } + + req := httptest.NewRequest("POST", "http://example.com/upload", body) + req.Header.Set("Content-Type", writer.FormDataContentType()) + + _, fileHeader, err := req.FormFile("documents") + if err != nil { + t.Fatalf("failed to parse form file: %v", err) + } + fileHeader.Header.Set("Content-Type", contentType) + return fileHeader +} diff --git a/internal/common/service/common.document.storage.go b/internal/common/service/common.document.storage.go new file mode 100644 index 00000000..24e6fade --- /dev/null +++ b/internal/common/service/common.document.storage.go @@ -0,0 +1,160 @@ +package service + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type DocumentStorage interface { + Upload(ctx context.Context, key string, body io.Reader, size int64, contentType string) (DocumentStorageUploadResult, error) + Delete(ctx context.Context, key string) error + URL(key string) string +} + +type DocumentStorageUploadResult struct { + Key string + URL string + ETag string +} + +type S3DocumentStorageConfig struct { + Region string + Bucket string + AccessKey string + SecretKey string + Endpoint string + BaseURL string + ForcePathStyle bool +} + +type s3DocumentStorage struct { + client *s3.Client + bucket string + base string +} + +func NewS3DocumentStorage(ctx context.Context, cfg S3DocumentStorageConfig) (DocumentStorage, error) { + bucket := strings.TrimSpace(cfg.Bucket) + if bucket == "" { + return nil, errors.New("s3 bucket is required") + } + region := strings.TrimSpace(cfg.Region) + if region == "" { + region = "us-east-1" + } + + options := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithRegion(region), + } + + endpoint := strings.TrimSpace(cfg.Endpoint) + if endpoint != "" { + resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, _ ...interface{}) (aws.Endpoint, error) { + if service == s3.ServiceID { + return aws.Endpoint{ + URL: endpoint, + SigningRegion: region, + HostnameImmutable: true, + }, nil + } + return aws.Endpoint{}, &aws.EndpointNotFoundError{} + }) + options = append(options, awsconfig.WithEndpointResolverWithOptions(resolver)) + } + + accessKey := strings.TrimSpace(cfg.AccessKey) + secretKey := strings.TrimSpace(cfg.SecretKey) + if accessKey != "" && secretKey != "" { + options = append(options, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""), + )) + } + + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, options...) + if err != nil { + return nil, err + } + + client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { + o.UsePathStyle = cfg.ForcePathStyle + }) + + baseURL := strings.TrimSuffix(strings.TrimSpace(cfg.BaseURL), "/") + if baseURL == "" { + if endpoint != "" { + baseURL = fmt.Sprintf("%s/%s", strings.TrimSuffix(endpoint, "/"), bucket) + } else { + baseURL = fmt.Sprintf("https://%s.s3.%s.amazonaws.com", bucket, region) + } + } + + return &s3DocumentStorage{ + client: client, + bucket: bucket, + base: baseURL, + }, nil +} + +func (s *s3DocumentStorage) Upload(ctx context.Context, key string, body io.Reader, size int64, contentType string) (DocumentStorageUploadResult, error) { + if strings.TrimSpace(key) == "" { + return DocumentStorageUploadResult{}, errors.New("storage key is required") + } + if size < 0 { + size = 0 + } + input := &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + Body: body, + } + input.ContentLength = aws.Int64(size) + if ct := strings.TrimSpace(contentType); ct != "" { + input.ContentType = aws.String(ct) + } + + out, err := s.client.PutObject(ctx, input) + if err != nil { + return DocumentStorageUploadResult{}, err + } + + var etag string + if out.ETag != nil { + etag = strings.Trim(*out.ETag, "\"") + } + + return DocumentStorageUploadResult{ + Key: key, + URL: s.URL(key), + ETag: etag, + }, nil +} + +func (s *s3DocumentStorage) Delete(ctx context.Context, key string) error { + if strings.TrimSpace(key) == "" { + return nil + } + _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + return err +} + +func (s *s3DocumentStorage) URL(key string) string { + key = strings.TrimPrefix(strings.TrimSpace(key), "/") + if key == "" { + return s.base + } + if s.base == "" { + return key + } + return fmt.Sprintf("%s/%s", s.base, key) +} diff --git a/internal/config/config.go b/internal/config/config.go index 2554bf57..5f76a9e0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -65,6 +65,14 @@ var ( SSOUserSyncDrift time.Duration SSOUserSyncNonceTTL time.Duration SSOUserSyncMaxBodyBytes int + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3ForcePathStyle bool + S3PublicBaseURL string + S3DocumentKeyPrefix string ) func init() { @@ -106,6 +114,16 @@ func init() { // Redis RedisURL = viper.GetString("REDIS_URL") + // Object storage + S3Endpoint = strings.TrimSpace(viper.GetString("S3_ENDPOINT")) + S3Region = strings.TrimSpace(viper.GetString("S3_REGION")) + S3Bucket = strings.TrimSpace(viper.GetString("S3_BUCKET")) + S3AccessKey = strings.TrimSpace(viper.GetString("S3_ACCESS_KEY")) + S3SecretKey = strings.TrimSpace(viper.GetString("S3_SECRET_KEY")) + S3ForcePathStyle = viper.GetBool("S3_FORCE_PATH_STYLE") + S3PublicBaseURL = strings.TrimSuffix(strings.TrimSpace(viper.GetString("S3_PUBLIC_BASE_URL")), "/") + S3DocumentKeyPrefix = defaultString(strings.Trim(strings.TrimSpace(viper.GetString("S3_DOCUMENT_PREFIX")), "/"), "docs") + // SSO integration SSOIssuer = viper.GetString("SSO_ISSUER") SSOJWKSURL = viper.GetString("SSO_JWKS_URL") diff --git a/internal/database/migrations/20251202103838_create_document_table.down.sql b/internal/database/migrations/20251202103838_create_document_table.down.sql new file mode 100644 index 00000000..68c3a98a --- /dev/null +++ b/internal/database/migrations/20251202103838_create_document_table.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS documents_documentable_polymorphic; +DROP TABLE IF EXISTS documents; diff --git a/internal/database/migrations/20251202103838_create_document_table.up.sql b/internal/database/migrations/20251202103838_create_document_table.up.sql new file mode 100644 index 00000000..cec686a4 --- /dev/null +++ b/internal/database/migrations/20251202103838_create_document_table.up.sql @@ -0,0 +1,14 @@ +CREATE TABLE documents ( + id BIGSERIAL PRIMARY KEY, + documentable_type VARCHAR(50) NOT NULL, + documentable_id BIGINT NOT NULL, + type VARCHAR(50) NOT NULL, + path VARCHAR(50) NOT NULL, + name VARCHAR(50) NOT NULL, + ext VARCHAR(50) NOT NULL, + size NUMERIC(15, 3) NOT NULL, + created_by BIGINT REFERENCES users (id) ON DELETE RESTRICT ON UPDATE CASCADE, + created_at TIMESTAMPTZ DEFAULT now() +); + +CREATE INDEX documents_documentable_polymorphic ON documents (documentable_type, documentable_id); diff --git a/internal/entities/document.go b/internal/entities/document.go new file mode 100644 index 00000000..54974a02 --- /dev/null +++ b/internal/entities/document.go @@ -0,0 +1,18 @@ +package entities + +import "time" + +type Document struct { + Id uint `gorm:"primaryKey"` + DocumentableType string `gorm:"size:50;not null;index:documents_documentable_polymorphic,priority:1"` + DocumentableId uint64 `gorm:"not null;index:documents_documentable_polymorphic,priority:2"` + Type string `gorm:"size:50;not null"` + Path string `gorm:"size:50;not null"` + Name string `gorm:"size:50;not null"` + Ext string `gorm:"size:50;not null"` + Size float64 `gorm:"type:numeric(15,3);not null"` + CreatedBy *uint `gorm:"index"` + CreatedAt time.Time `gorm:"autoCreateTime"` + + CreatedUser *User `gorm:"foreignKey:CreatedBy;references:Id"` +}