
Product
Secure Your AI-Generated Code with Socket MCP
Socket MCP brings real-time security checks to AI-generated code, helping developers catch risky dependencies before they enter the codebase.
github.com/beyondstorage/go-storage/v4
A vendor-neutral storage library for Golang.
Write once, run on every storage service.
package main
import (
"log"
"github.com/beyondstorage/go-storage/v4/services"
"github.com/beyondstorage/go-storage/v4/types"
// Add fs support
_ "github.com/beyondstorage/go-service-fs/v3"
// Add s3 support
_ "github.com/beyondstorage/go-service-s3/v2"
// Add gcs support
_ "github.com/beyondstorage/go-service-gcs/v2"
// Add azblob support
_ "github.com/beyondstorage/go-service-azblob/v2"
// More support could be found under BeyondStorage.
_ "github.com/beyondstorage/go-service-xxx"
)
func main() {
// Init a Storager from connection string.
store, err := services.NewStoragerFromString("s3://bucket_name/path/to/workdir")
if err != nil {
log.Fatalf("service init failed: %v", err)
}
// Write data from io.Reader into hello.txt
n, err := store.Write("hello.txt", r, length)
// Read data from hello.txt to io.Writer
n, err := store.Read("hello.txt", w)
// Stat hello.txt to check existence or get its metadata
o, err := store.Stat("hello.txt")
// Use object's functions to get metadata
length, ok := o.GetContentLength()
// List will create an iterator of object under path.
it, err := store.List("path")
for {
// Use iterator.Next to retrieve next object until we meet IteratorDone.
o, err := it.Next()
if errors.Is(err, types.IteraoorDone) {
break
}
}
// Delete hello.txt
err = store.Delete("hello.txt")
}
More examples could be found at go-storage-example.
16 stable services that have passed all integration tests.
3 beta services that implemented required functions, but not passed integration tests.
4 alpha services that still under development.
More service ideas could be found at Service Integration Tracking.
Basic operations
Storager
metadatameta := store.Metadata()
_ := meta.GetWorkDir() // Get object WorkDir
_, ok := meta.GetWriteSizeMaximum() // Get the maximum size for write operation
Object
content// Read 2048 byte at the offset 1024 into the io.Writer.
n, err := store.Read("path", w, pairs.WithOffset(1024), pairs.WithSize(2048))
Object
// Write 2048 byte from io.Reader
n, err := store.Write("path", r, 2048)
Object
metadata or check existenceso, err := store.Stat("path")
if errors.Is(err, services.ErrObjectNotExist) {
// object is not exist
}
length, ok := o.GetContentLength() // get the object content length.
Object
err := store.Delete("path") // Delete the object "path"
Object
in given prefix or dirit, err := store.List("path")
for {
o, err := it.Next()
if err != nil && errors.Is(err, types.IteratorDone) {
// the list is over
}
length, ok := o.GetContentLength() // get the object content length.
}
Extended operations
Object
inside storagererr := store.(Copier).Copy(src, dst) // Copy an object from src to dst.
Object
inside storagererr := store.(Mover).Move(src, dst) // Move an object from src to dst.
Object
url, err := store.(Reacher).Reach("path") // Generate an url to the object.
Object
supporto, err := store.(Direr).CreateDir("path") // Create a dir object.
Large file manipulation
ms := store.(Multiparter)
// Create a multipart object.
o, err := ms.CreateMultipart("path")
// Write 1024 bytes from io.Reader into a multipart at index 1
n, part, err := ms.WriteMultipart(o, r, 1024, 1)
// Complete a multipart object.
err := ms.CompleteMultipart(o, []*Part{part})
as := store.(Appender)
// Create an appendable object.
o, err := as.CreateAppend("path")
// Write 1024 bytes from io.Reader.
n, err := as.WriteAppend(o, r, 1024)
// Commit an append object.
err = as.CommitAppend(o)
bs := store.(Blocker)
// Create a block object.
o, err := bs.CreateBlock("path")
// Write 1024 bytes from io.Reader with block id "id-abc"
n, err := bs.WriteBlock(o, r, 1024, "id-abc")
// Combine block via block ids.
err := bs.CombineBlock(o, []string{"id-abc"})
ps := store.(Pager)
// Create a page object.
o, err := ps.CreatePage("path")
// Write 1024 bytes from io.Reader at offset 2048
n, err := ps.WritePage(o, r, 1024, 2048)
Global object metadata
id
: unique key in servicename
: relative path towards service's work dirmode
: object mode can be a combination of read
, dir
, part
and moreetag
: entity tag as defined in rfc2616content-length
: object's content size.content-md5
: md5 digest as defined in rfc2616content-type
: media type as defined in rfc2616last-modified
: object's last updated time.System object metadata
Service system object metadata like storage-class
and so on.
o, err := store.Stat("path")
// Get service system metadata via API provides by go-service-s3.
om := s3.GetObjectSystemMetadata(o)
_ = om.StorageClass // this object's storage class
_ = om.ServerSideEncryptionCustomerAlgorithm // this object's sse algorithm
Self maintained codegen definitions helps to generate all our APIs, pairs and metadata.
Generated pairs which can be used as API optional arguments.
func WithContentMd5(v string) Pair {
return Pair{
Key: "content_md5",
Value: v,
}
}
Generated object metadata which can be used to get content md5 from object.
func (o *Object) GetContentMd5() (string, bool) {
o.stat()
if o.bit&objectIndexContentMd5 != 0 {
return o.contentMd5, true
}
return "", false
}
Server-Side Encrypt supports via system pair and system metadata, and we can use Default Pairs to simplify the job.
func NewS3SseC(key []byte) (types.Storager, error) {
defaultPairs := s3.DefaultStoragePairs{
Write: []types.Pair{
// Required, must be AES256
s3.WithServerSideEncryptionCustomerAlgorithm(s3.ServerSideEncryptionAes256),
// Required, your AES-256 key, a 32-byte binary value
s3.WithServerSideEncryptionCustomerKey(key),
},
// Now you have to provide customer key to read encrypted data
Read: []types.Pair{
// Required, must be AES256
s3.WithServerSideEncryptionCustomerAlgorithm(s3.ServerSideEncryptionAes256),
// Required, your AES-256 key, a 32-byte binary value
s3.WithServerSideEncryptionCustomerKey(key),
}}
return s3.NewStorager(..., s3.WithDefaultStoragePairs(defaultPairs))
}
FAQs
Unknown package
Did you know?
Socket for GitHub automatically highlights issues in each pull request and monitors the health of all your open source dependencies. Discover the contents of your packages and block harmful activity before you install or update your dependencies.
Product
Socket MCP brings real-time security checks to AI-generated code, helping developers catch risky dependencies before they enter the codebase.
Security News
As vulnerability data bottlenecks grow, the federal government is formally investigating NIST’s handling of the National Vulnerability Database.
Research
Security News
Socket’s Threat Research Team has uncovered 60 npm packages using post-install scripts to silently exfiltrate hostnames, IP addresses, DNS servers, and user directories to a Discord-controlled endpoint.