Merge branch 'dev/hafizh' into 'feat/BE/Sprint-6'

feat(BE-308,309): utility document and implementation s3 bucket

See merge request mbugroup/lti-api!78
This commit is contained in:
Hafizh A. Y.
2025-12-04 08:46:50 +00:00
10 changed files with 844 additions and 1 deletions
@@ -0,0 +1,62 @@
package repository
import (
"context"
entity "gitlab.com/mbugroup/lti-api.git/internal/entities"
"gorm.io/gorm"
)
type DocumentRepository interface {
BaseRepository[entity.Document]
ListByTarget(ctx context.Context, documentableType string, documentableID uint64, modifier func(*gorm.DB) *gorm.DB) ([]entity.Document, error)
DeleteByTarget(ctx context.Context, documentableType string, documentableID uint64, modifier func(*gorm.DB) *gorm.DB) error
}
type documentRepositoryImpl struct {
*BaseRepositoryImpl[entity.Document]
}
func NewDocumentRepository(db *gorm.DB) DocumentRepository {
return &documentRepositoryImpl{
BaseRepositoryImpl: NewBaseRepository[entity.Document](db),
}
}
func (r *documentRepositoryImpl) ListByTarget(
ctx context.Context,
documentableType string,
documentableID uint64,
modifier func(*gorm.DB) *gorm.DB,
) ([]entity.Document, error) {
var documents []entity.Document
q := r.DB().WithContext(ctx).
Where("documentable_type = ? AND documentable_id = ?", documentableType, documentableID)
if modifier != nil {
q = modifier(q)
}
if err := q.Order("created_at ASC").Find(&documents).Error; err != nil {
return nil, err
}
return documents, nil
}
func (r *documentRepositoryImpl) DeleteByTarget(
ctx context.Context,
documentableType string,
documentableID uint64,
modifier func(*gorm.DB) *gorm.DB,
) error {
q := r.DB().WithContext(ctx).
Where("documentable_type = ? AND documentable_id = ?", documentableType, documentableID)
if modifier != nil {
q = modifier(q)
}
return q.Delete(&entity.Document{}).Error
}
@@ -0,0 +1,411 @@
package service
import (
"context"
"errors"
"fmt"
"mime"
"mime/multipart"
"path/filepath"
"strings"
commonRepo "gitlab.com/mbugroup/lti-api.git/internal/common/repository"
"gitlab.com/mbugroup/lti-api.git/internal/config"
entity "gitlab.com/mbugroup/lti-api.git/internal/entities"
"gitlab.com/mbugroup/lti-api.git/internal/utils"
"github.com/google/uuid"
)
const (
defaultDocumentPathLimit = 50
defaultDocumentKeyPrefix = "docs"
maxDocumentNameLength = 50
)
type DocumentService interface {
UploadDocuments(ctx context.Context, req DocumentUploadRequest) ([]DocumentUploadResult, error)
ListByTarget(ctx context.Context, documentableType string, documentableID uint64) ([]entity.Document, error)
DeleteDocuments(ctx context.Context, ids []uint, removeFromStorage bool) error
DeleteByTarget(ctx context.Context, documentableType string, documentableID uint64, removeFromStorage bool) error
PublicURL(document entity.Document) string
}
type DocumentUploadRequest struct {
DocumentableType string
DocumentableID uint64
CreatedBy *uint
Files []DocumentFile
}
type DocumentFile struct {
File *multipart.FileHeader
Type string
Index *int
}
type DocumentUploadResult struct {
Document entity.Document
URL string
Index *int
}
type DocumentServiceOption func(*documentService)
type documentService struct {
repo commonRepo.DocumentRepository
storage DocumentStorage
keyPrefix string
maxPathLength int
}
func NewDocumentService(repo commonRepo.DocumentRepository, storage DocumentStorage, opts ...DocumentServiceOption) DocumentService {
svc := &documentService{
repo: repo,
storage: storage,
keyPrefix: defaultDocumentKeyPrefix,
maxPathLength: defaultDocumentPathLimit,
}
for _, opt := range opts {
opt(svc)
}
return svc
}
func NewDocumentServiceFromConfig(ctx context.Context, repo commonRepo.DocumentRepository) (DocumentService, error) {
if repo == nil {
return nil, errors.New("document repository is required")
}
if strings.TrimSpace(config.S3Bucket) == "" {
return nil, errors.New("S3_BUCKET is not configured")
}
storage, err := NewS3DocumentStorage(ctx, S3DocumentStorageConfig{
Region: config.S3Region,
Bucket: config.S3Bucket,
AccessKey: config.S3AccessKey,
SecretKey: config.S3SecretKey,
Endpoint: config.S3Endpoint,
BaseURL: config.S3PublicBaseURL,
ForcePathStyle: config.S3ForcePathStyle,
})
if err != nil {
return nil, err
}
prefix := config.S3DocumentKeyPrefix
if prefix == "" {
prefix = defaultDocumentKeyPrefix
}
return NewDocumentService(
repo,
storage,
WithDocumentKeyPrefix(prefix),
WithDocumentPathLimit(defaultDocumentPathLimit),
), nil
}
func WithDocumentKeyPrefix(prefix string) DocumentServiceOption {
return func(svc *documentService) {
prefix = strings.Trim(prefix, "/")
if prefix == "" {
prefix = defaultDocumentKeyPrefix
}
svc.keyPrefix = prefix
}
}
func WithDocumentPathLimit(limit int) DocumentServiceOption {
return func(svc *documentService) {
if limit > 0 {
svc.maxPathLength = limit
}
}
}
func (s *documentService) UploadDocuments(ctx context.Context, req DocumentUploadRequest) ([]DocumentUploadResult, error) {
if s.repo == nil {
return nil, errors.New("document repository not configured")
}
if s.storage == nil {
return nil, errors.New("document storage not configured")
}
documentableType := strings.ToUpper(strings.TrimSpace(req.DocumentableType))
if documentableType == "" {
return nil, errors.New("documentable type is required")
}
if req.DocumentableID == 0 {
return nil, errors.New("documentable id is required")
}
if len(req.Files) == 0 {
return nil, errors.New("no files to upload")
}
var createdBy *uint
if req.CreatedBy != nil && *req.CreatedBy != 0 {
idCopy := *req.CreatedBy
createdBy = &idCopy
}
results := make([]DocumentUploadResult, 0, len(req.Files))
createdDocs := make([]entity.Document, 0, len(req.Files))
for _, file := range req.Files {
if file.File == nil {
return nil, errors.New("file header is required")
}
originalName := sanitizeDocumentName(file.File.Filename)
contentType := detectContentType(file.File, originalName)
ext := detectExtension(file.File.Filename, contentType)
key, err := s.generateObjectKey(ext)
if err != nil {
s.rollbackDocuments(ctx, createdDocs)
return nil, err
}
reader, err := file.File.Open()
if err != nil {
s.rollbackDocuments(ctx, createdDocs)
return nil, err
}
uploadRes, err := s.storage.Upload(ctx, key, reader, file.File.Size, contentType)
_ = reader.Close()
if err != nil {
s.rollbackDocuments(ctx, createdDocs)
return nil, err
}
docType := resolveDocumentType(file.Type, documentableType)
doc := entity.Document{
DocumentableType: documentableType,
DocumentableId: req.DocumentableID,
Type: docType,
Path: uploadRes.Key,
Name: originalName,
Ext: strings.TrimPrefix(ext, "."),
Size: float64(file.File.Size),
CreatedBy: createdBy,
}
if err := s.repo.CreateOne(ctx, &doc, nil); err != nil {
_ = s.storage.Delete(ctx, uploadRes.Key)
s.rollbackDocuments(ctx, createdDocs)
return nil, err
}
createdDocs = append(createdDocs, doc)
results = append(results, DocumentUploadResult{
Document: doc,
URL: uploadRes.URL,
Index: cloneIndex(file.Index),
})
}
return results, nil
}
func (s *documentService) ListByTarget(ctx context.Context, documentableType string, documentableID uint64) ([]entity.Document, error) {
if s.repo == nil {
return nil, errors.New("document repository not configured")
}
documentableType = strings.ToUpper(strings.TrimSpace(documentableType))
if documentableType == "" {
return nil, errors.New("documentable type is required")
}
if documentableID == 0 {
return nil, errors.New("documentable id is required")
}
return s.repo.ListByTarget(ctx, documentableType, documentableID, nil)
}
func (s *documentService) DeleteDocuments(ctx context.Context, ids []uint, removeFromStorage bool) error {
if s.repo == nil {
return errors.New("document repository not configured")
}
if len(ids) == 0 {
return nil
}
docs, err := s.repo.GetByIDs(ctx, ids, nil)
if err != nil {
return err
}
for _, doc := range docs {
if err := s.repo.DeleteOne(ctx, doc.Id); err != nil {
return err
}
if removeFromStorage && s.storage != nil {
if err := s.storage.Delete(ctx, doc.Path); err != nil {
utils.Log.WithError(err).Warnf("failed to delete document object %s", doc.Path)
}
}
}
return nil
}
func (s *documentService) DeleteByTarget(ctx context.Context, documentableType string, documentableID uint64, removeFromStorage bool) error {
if s.repo == nil {
return errors.New("document repository not configured")
}
documentableType = strings.ToUpper(strings.TrimSpace(documentableType))
if documentableType == "" || documentableID == 0 {
return errors.New("documentable type and id are required")
}
var docs []entity.Document
if removeFromStorage && s.storage != nil {
var err error
docs, err = s.repo.ListByTarget(ctx, documentableType, documentableID, nil)
if err != nil {
return err
}
}
if err := s.repo.DeleteByTarget(ctx, documentableType, documentableID, nil); err != nil {
return err
}
if removeFromStorage && len(docs) > 0 {
for _, doc := range docs {
if err := s.storage.Delete(ctx, doc.Path); err != nil {
utils.Log.WithError(err).Warnf("failed to delete document object %s", doc.Path)
}
}
}
return nil
}
func (s *documentService) PublicURL(document entity.Document) string {
if s.storage == nil || strings.TrimSpace(document.Path) == "" {
return ""
}
return s.storage.URL(document.Path)
}
func (s *documentService) generateObjectKey(ext string) (string, error) {
normalizedExt := strings.TrimSpace(ext)
if normalizedExt != "" && !strings.HasPrefix(normalizedExt, ".") {
normalizedExt = "." + normalizedExt
}
u := uuid.New().String()
key := fmt.Sprintf("%s/%s%s", strings.Trim(s.keyPrefix, "/"), u, normalizedExt)
if s.keyPrefix == "" {
key = fmt.Sprintf("%s%s", u, normalizedExt)
}
if len(key) > s.maxPathLength {
key = fmt.Sprintf("%s%s", u, normalizedExt)
}
if len(key) > s.maxPathLength {
return "", fmt.Errorf("object key exceeds maximum length (%d)", s.maxPathLength)
}
return key, nil
}
func (s *documentService) rollbackDocuments(ctx context.Context, docs []entity.Document) {
if len(docs) == 0 {
return
}
for i := len(docs) - 1; i >= 0; i-- {
doc := docs[i]
if s.repo != nil && doc.Id != 0 {
if err := s.repo.DeleteOne(ctx, doc.Id); err != nil {
utils.Log.WithError(err).Warnf("failed to rollback document #%d", doc.Id)
}
}
if s.storage != nil && strings.TrimSpace(doc.Path) != "" {
if err := s.storage.Delete(ctx, doc.Path); err != nil {
utils.Log.WithError(err).Warnf("failed to rollback document object %s", doc.Path)
}
}
}
}
func sanitizeDocumentName(name string) string {
name = filepath.Base(strings.TrimSpace(name))
if name == "." || name == "" {
name = "document"
}
name = strings.Map(func(r rune) rune {
if r < 32 {
return -1
}
switch r {
case '\\', '/', ':', '*', '?', '"', '<', '>', '|':
return '-'
default:
return r
}
}, name)
if len(name) > maxDocumentNameLength {
runes := []rune(name)
if len(runes) > maxDocumentNameLength {
name = string(runes[:maxDocumentNameLength])
}
}
return name
}
func detectExtension(filename, contentType string) string {
ext := strings.ToLower(strings.TrimSpace(filepath.Ext(filename)))
if ext == "" && contentType != "" {
if exts, _ := mime.ExtensionsByType(contentType); len(exts) > 0 {
ext = exts[0]
}
}
if ext == "" {
return ".bin"
}
if !strings.HasPrefix(ext, ".") {
ext = "." + ext
}
return ext
}
func detectContentType(file *multipart.FileHeader, filename string) string {
if file == nil {
return "application/octet-stream"
}
contentType := strings.TrimSpace(file.Header.Get("Content-Type"))
if contentType != "" {
return contentType
}
if ext := filepath.Ext(filename); ext != "" {
if guess := mime.TypeByExtension(ext); guess != "" {
return guess
}
}
return "application/octet-stream"
}
func resolveDocumentType(fileType, fallback string) string {
value := strings.ToUpper(strings.TrimSpace(fileType))
if value == "" {
return fallback
}
return value
}
func cloneIndex(index *int) *int {
if index == nil {
return nil
}
value := *index
return &value
}
@@ -0,0 +1,101 @@
package service
import (
"bytes"
"context"
"mime/multipart"
"net/http/httptest"
"strings"
"testing"
commonRepo "gitlab.com/mbugroup/lti-api.git/internal/common/repository"
"gitlab.com/mbugroup/lti-api.git/internal/config"
"gitlab.com/mbugroup/lti-api.git/internal/database"
entity "gitlab.com/mbugroup/lti-api.git/internal/entities"
"gorm.io/gorm"
)
func TestDocumentServiceUpload(t *testing.T) {
if strings.TrimSpace(config.S3Bucket) == "" {
t.Fatal("S3 bucket is not configured; set S3_* env vars to run this test")
}
ctx := context.Background()
db := setupDocumentTestDB(t)
repo := commonRepo.NewDocumentRepository(db)
svc, err := NewDocumentServiceFromConfig(ctx, repo)
if err != nil {
t.Fatalf("failed to create document service from config: %v", err)
}
file := newTestFileHeader(t, "integration-proof.txt", "text/plain", []byte("document integration test"))
userID := uint(100)
results, err := svc.UploadDocuments(ctx, DocumentUploadRequest{
DocumentableType: "INVENTORY_TRANSFER",
DocumentableID: 99,
CreatedBy: &userID,
Files: []DocumentFile{
{File: file, Type: "integration"},
},
})
if err != nil {
t.Fatalf("upload to S3 failed: %v", err)
}
if len(results) != 1 {
t.Fatalf("expected 1 uploaded document, got %d", len(results))
}
doc := results[0].Document
if doc.Path == "" {
t.Fatalf("expected non-empty storage path")
}
if results[0].URL == "" {
t.Fatalf("expected public URL for uploaded document")
}
t.Logf("uploaded document #%d to %s (path=%s)", doc.Id, results[0].URL, doc.Path)
}
func setupDocumentTestDB(t *testing.T) *gorm.DB {
t.Helper()
if strings.TrimSpace(config.DBHost) == "" || strings.TrimSpace(config.DBName) == "" {
t.Fatal("database configuration missing; ensure DB_HOST and DB_NAME are set")
}
db := database.Connect(config.DBHost, config.DBName)
if db == nil {
t.Fatal("failed to create database connection")
}
if err := db.AutoMigrate(&entity.Document{}); err != nil {
t.Fatalf("failed to migrate document table: %v", err)
}
return db
}
func newTestFileHeader(t *testing.T, filename, contentType string, data []byte) *multipart.FileHeader {
t.Helper()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("documents", filename)
if err != nil {
t.Fatalf("failed to create form file: %v", err)
}
if _, err := part.Write(data); err != nil {
t.Fatalf("failed to write file data: %v", err)
}
if err := writer.Close(); err != nil {
t.Fatalf("failed to close writer: %v", err)
}
req := httptest.NewRequest("POST", "http://example.com/upload", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
_, fileHeader, err := req.FormFile("documents")
if err != nil {
t.Fatalf("failed to parse form file: %v", err)
}
fileHeader.Header.Set("Content-Type", contentType)
return fileHeader
}
@@ -0,0 +1,160 @@
package service
import (
"context"
"errors"
"fmt"
"io"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
type DocumentStorage interface {
Upload(ctx context.Context, key string, body io.Reader, size int64, contentType string) (DocumentStorageUploadResult, error)
Delete(ctx context.Context, key string) error
URL(key string) string
}
type DocumentStorageUploadResult struct {
Key string
URL string
ETag string
}
type S3DocumentStorageConfig struct {
Region string
Bucket string
AccessKey string
SecretKey string
Endpoint string
BaseURL string
ForcePathStyle bool
}
type s3DocumentStorage struct {
client *s3.Client
bucket string
base string
}
func NewS3DocumentStorage(ctx context.Context, cfg S3DocumentStorageConfig) (DocumentStorage, error) {
bucket := strings.TrimSpace(cfg.Bucket)
if bucket == "" {
return nil, errors.New("s3 bucket is required")
}
region := strings.TrimSpace(cfg.Region)
if region == "" {
region = "us-east-1"
}
options := []func(*awsconfig.LoadOptions) error{
awsconfig.WithRegion(region),
}
endpoint := strings.TrimSpace(cfg.Endpoint)
if endpoint != "" {
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, _ ...interface{}) (aws.Endpoint, error) {
if service == s3.ServiceID {
return aws.Endpoint{
URL: endpoint,
SigningRegion: region,
HostnameImmutable: true,
}, nil
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
options = append(options, awsconfig.WithEndpointResolverWithOptions(resolver))
}
accessKey := strings.TrimSpace(cfg.AccessKey)
secretKey := strings.TrimSpace(cfg.SecretKey)
if accessKey != "" && secretKey != "" {
options = append(options, awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
))
}
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, options...)
if err != nil {
return nil, err
}
client := s3.NewFromConfig(awsCfg, func(o *s3.Options) {
o.UsePathStyle = cfg.ForcePathStyle
})
baseURL := strings.TrimSuffix(strings.TrimSpace(cfg.BaseURL), "/")
if baseURL == "" {
if endpoint != "" {
baseURL = fmt.Sprintf("%s/%s", strings.TrimSuffix(endpoint, "/"), bucket)
} else {
baseURL = fmt.Sprintf("https://%s.s3.%s.amazonaws.com", bucket, region)
}
}
return &s3DocumentStorage{
client: client,
bucket: bucket,
base: baseURL,
}, nil
}
func (s *s3DocumentStorage) Upload(ctx context.Context, key string, body io.Reader, size int64, contentType string) (DocumentStorageUploadResult, error) {
if strings.TrimSpace(key) == "" {
return DocumentStorageUploadResult{}, errors.New("storage key is required")
}
if size < 0 {
size = 0
}
input := &s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
Body: body,
}
input.ContentLength = aws.Int64(size)
if ct := strings.TrimSpace(contentType); ct != "" {
input.ContentType = aws.String(ct)
}
out, err := s.client.PutObject(ctx, input)
if err != nil {
return DocumentStorageUploadResult{}, err
}
var etag string
if out.ETag != nil {
etag = strings.Trim(*out.ETag, "\"")
}
return DocumentStorageUploadResult{
Key: key,
URL: s.URL(key),
ETag: etag,
}, nil
}
func (s *s3DocumentStorage) Delete(ctx context.Context, key string) error {
if strings.TrimSpace(key) == "" {
return nil
}
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
})
return err
}
func (s *s3DocumentStorage) URL(key string) string {
key = strings.TrimPrefix(strings.TrimSpace(key), "/")
if key == "" {
return s.base
}
if s.base == "" {
return key
}
return fmt.Sprintf("%s/%s", s.base, key)
}
+18
View File
@@ -65,6 +65,14 @@ var (
SSOUserSyncDrift time.Duration
SSOUserSyncNonceTTL time.Duration
SSOUserSyncMaxBodyBytes int
S3Endpoint string
S3Region string
S3Bucket string
S3AccessKey string
S3SecretKey string
S3ForcePathStyle bool
S3PublicBaseURL string
S3DocumentKeyPrefix string
)
func init() {
@@ -106,6 +114,16 @@ func init() {
// Redis
RedisURL = viper.GetString("REDIS_URL")
// Object storage
S3Endpoint = strings.TrimSpace(viper.GetString("S3_ENDPOINT"))
S3Region = strings.TrimSpace(viper.GetString("S3_REGION"))
S3Bucket = strings.TrimSpace(viper.GetString("S3_BUCKET"))
S3AccessKey = strings.TrimSpace(viper.GetString("S3_ACCESS_KEY"))
S3SecretKey = strings.TrimSpace(viper.GetString("S3_SECRET_KEY"))
S3ForcePathStyle = viper.GetBool("S3_FORCE_PATH_STYLE")
S3PublicBaseURL = strings.TrimSuffix(strings.TrimSpace(viper.GetString("S3_PUBLIC_BASE_URL")), "/")
S3DocumentKeyPrefix = defaultString(strings.Trim(strings.TrimSpace(viper.GetString("S3_DOCUMENT_PREFIX")), "/"), "docs")
// SSO integration
SSOIssuer = viper.GetString("SSO_ISSUER")
SSOJWKSURL = viper.GetString("SSO_JWKS_URL")
@@ -0,0 +1,2 @@
DROP INDEX IF EXISTS documents_documentable_polymorphic;
DROP TABLE IF EXISTS documents;
@@ -0,0 +1,14 @@
CREATE TABLE documents (
id BIGSERIAL PRIMARY KEY,
documentable_type VARCHAR(50) NOT NULL,
documentable_id BIGINT NOT NULL,
type VARCHAR(50) NOT NULL,
path VARCHAR(50) NOT NULL,
name VARCHAR(50) NOT NULL,
ext VARCHAR(50) NOT NULL,
size NUMERIC(15, 3) NOT NULL,
created_by BIGINT REFERENCES users (id) ON DELETE RESTRICT ON UPDATE CASCADE,
created_at TIMESTAMPTZ DEFAULT now()
);
CREATE INDEX documents_documentable_polymorphic ON documents (documentable_type, documentable_id);
+18
View File
@@ -0,0 +1,18 @@
package entities
import "time"
type Document struct {
Id uint `gorm:"primaryKey"`
DocumentableType string `gorm:"size:50;not null;index:documents_documentable_polymorphic,priority:1"`
DocumentableId uint64 `gorm:"not null;index:documents_documentable_polymorphic,priority:2"`
Type string `gorm:"size:50;not null"`
Path string `gorm:"size:50;not null"`
Name string `gorm:"size:50;not null"`
Ext string `gorm:"size:50;not null"`
Size float64 `gorm:"type:numeric(15,3);not null"`
CreatedBy *uint `gorm:"index"`
CreatedAt time.Time `gorm:"autoCreateTime"`
CreatedUser *User `gorm:"foreignKey:CreatedBy;references:Id"`
}