Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(hash): support sha256 verification #5

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 14 additions & 13 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# oget

oget is a Golang download library. It supports parallel downloads, resuming after failures, SHA512 verification, and download progress monitoring.
oget is a Golang download library. It supports parallel downloads, resuming after failures, Hash verification, and download progress monitoring.

## Installation

Expand Down Expand Up @@ -83,22 +83,23 @@ if err != nil {
}
```

### SHA512 Verification
### Hash Verification

After downloading, the library performs a SHA512 checksum on the entire file. If the checksum fails, an `oget.SHA512Error` is thrown.
After downloading, the library performs a Hash checksum on the entire file. If the checksum fails, an `oget.HashError` is thrown.

```go
import "github.com/oomol-lab/oget"

_, err := (&OGet{
_, err := (&oget.OGet{
URL: "https://github.com/oomol-lab/oget/raw/main/tests/target.bin",
FilePath: "/path/to/save/file.bin",
SHA512: "d286fbb1fab9014fdbc543d09f54cb93da6e0f2c809e62ee0c81d69e4bf58eec44571fae192a8da9bc772ce1340a0d51ad638cdba6118909b555a12b005f2930",
HashType: oget.SHA512,
Hash: "d286fbb1fab9014fdbc543d09f54cb93da6e0f2c809e62ee0c81d69e4bf58eec44571fae192a8da9bc772ce1340a0d51ad638cdba6118909b555a12b005f2930",
}).Get()

if err != nil {
if sha512Error, ok := err.(oget.SHA512Error); ok {
// Failed due to SHA512 verification failure
if hashError, ok := err.(oget.HashError); ok {
// Failed due to hash verification failure
}
panic(err)
}
Expand All @@ -113,15 +114,15 @@ import "github.com/oomol-lab/oget"
success := false

for i := 0; i < 10; i++ {
clean, err := (&OGet{
clean, err := (&oget.OGet{
URL: "https://github.com/oomol-lab/oget/raw/main/tests/target.bin",
FilePath: "/path/to/save/file.bin",
Parts: 4,
}).Get()
if err != nil {
if sha512Error, ok := err.(oget.SHA512Error); ok {
if hashError, ok := err.(oget.HashError); ok {
clean()
panic(sha512Error)
panic(hashError)
}
fmt.Printf("download failed with error and retry %s", err)
} else {
Expand All @@ -147,7 +148,7 @@ if err != nil {
}
```

Then, call `task.Get()` to initiate the download. Check if the error is of type `oget.SHA512Error`. If not, it is likely due to network issues and should be retried.
Then, call `task.Get()` to initiate the download. Check if the error is of type `oget.HashError`. If not, it is likely due to network issues and should be retried.

Note that the first return value of `task.Get()` is a function `clean` that deletes the temporary download files. Call it to free up disk space if you don't want to keep these files for the next download attempt after a download failure.

Expand All @@ -160,9 +161,9 @@ for i := 0; i < 10; i++ {
Parts: 4,
})
if err != nil {
if sha512Error, ok := err.(oget.SHA512Error); ok {
if hashError, ok := err.(oget.HashError); ok {
clean()
panic(sha512Error)
panic(hashError)
}
fmt.Printf("download failed with error and retry %s", err)
} else {
Expand Down
7 changes: 4 additions & 3 deletions config.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,10 @@ type RemoteFile struct {
type GettingConfig struct {
// the path to save the downloaded file.
FilePath string
// the SHA512 code of the file.
// if the code is empty, the file will not be checked.
SHA512 string
// the Hash code of the file.
// if the hash is empty, the file will not be checked.
HashType HashType
Hash string
// PartsPath is the path to save the temp files of downloaded parts.
// if the value is empty, the temp files will be saved in the same directory as the FilePath.
PartsPath string
Expand Down
10 changes: 6 additions & 4 deletions get.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ type OGet struct {
// the maximum number of idle (keep-alive) connections to keep per-host.
// the default is 16.
MaxIdleConnsPerHost int
// the SHA512 code of the file.
// if the code is empty, the file will not be checked.
SHA512 string
// the Hash code of the file.
// if the hash is empty, the file will not be checked.
HashType HashType
Hash string
// PartsPath is the path to save the temp files of downloaded parts.
// if the value is empty, the temp files will be saved in the same directory as the FilePath.
PartsPath string
Expand Down Expand Up @@ -57,7 +58,8 @@ func (o *OGet) Get() (func() error, error) {
}
return task.Get(&GettingConfig{
FilePath: o.FilePath,
SHA512: o.SHA512,
HashType: o.HashType,
Hash: o.Hash,
PartsPath: o.PartsPath,
PartName: o.PartName,
Parts: o.Parts,
Expand Down
61 changes: 61 additions & 0 deletions hash.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package oget

import (
"crypto/sha256"
"crypto/sha512"
"fmt"
"hash"
"io"
"os"
)

type HashType string

const (
SHA256 HashType = "sha256"
SHA512 HashType = "sha512"
)

func supportHashType(s HashType) bool {
return s == SHA256 || s == SHA512
}

func SHA(path string, ht HashType) (string, error) {
return shaOfFiles(&[]string{path}, ht)
}

func shaOfFiles(pathList *[]string, ht HashType) (string, error) {
var h hash.Hash

switch ht {
case SHA256:
h = sha256.New()
case SHA512:
h = sha512.New()
}

for _, path := range *pathList {
file, err := os.Open(path)
if err != nil {
return "", err
}
defer file.Close()
io.Copy(h, file)
}
hashInBytes := h.Sum(nil)
hashStr := fmt.Sprintf("%x", hashInBytes)

return hashStr, nil
}

type HashError struct {
message string
}

func (e HashError) Error() string {
return e.message
}

func createHashError(message string) HashError {
return HashError{message}
}
42 changes: 0 additions & 42 deletions sha512.go

This file was deleted.

14 changes: 7 additions & 7 deletions task.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,8 @@ import (
"mime"
"net/http"
"os"
"time"

"path/filepath"
"time"

"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
Expand Down Expand Up @@ -201,15 +200,16 @@ func (t *GettingTask) mergeFile(c *GettingConfig, prog *progress) error {
partPath := filepath.Join(c.PartsPath, c.partFileName(i))
partPathList = append(partPathList, partPath)
}
if c.SHA512 != "" {
code, err := sha512OfFiles(&partPathList)
if c.Hash != "" && supportHashType(c.HashType) {
code, err := shaOfFiles(&partPathList, c.HashType)
if err != nil {
return errors.Wrapf(err, "failed to get sha512 code")
return errors.Wrapf(err, fmt.Sprintf("failed to get %s code", c.HashType))
}
if code != c.SHA512 {
return createSHA512Error("sha512 code does not match")
if code != c.Hash {
return createHashError(fmt.Sprintf("%s code does not match", c.HashType))
}
}

if len(partPathList) == 1 {
partPath := partPathList[0]
err := os.Rename(partPath, c.FilePath)
Expand Down
14 changes: 9 additions & 5 deletions tests/oget_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ func TestAll(t *testing.T) {
_, err = task.Get(&oget.GettingConfig{
FilePath: filepath.Join(outputPath, "target.bin"),
PartsPath: partsPath,
SHA512: sha512Code,
HashType: oget.SHA512,
Hash: sha512Code,
})
if err != nil {
t.Fatalf("download file: %s", err)
Expand All @@ -70,12 +71,13 @@ func TestAll(t *testing.T) {
FilePath: savedFilePath,
PartsPath: partsPath,
Parts: 4,
SHA512: sha512Code,
HashType: oget.SHA512,
Hash: sha512Code,
})
if err != nil {
t.Fatalf("download file: %s", err)
}
savedFileCode, err := oget.SHA512(savedFilePath)
savedFileCode, err := oget.SHA(savedFilePath, oget.SHA512)

if err != nil {
t.Fatalf("get code of sha512 fail: %s", err)
Expand All @@ -100,7 +102,8 @@ func TestAll(t *testing.T) {
FilePath: savedFilePath,
PartsPath: partsPath,
Parts: 4,
SHA512: sha512Code,
HashType: oget.SHA512,
Hash: sha512Code,
ListenProgress: func(event oget.ProgressEvent) {
mux.Lock()
events = append(events, event)
Expand Down Expand Up @@ -157,7 +160,8 @@ func TestAll(t *testing.T) {
FilePath: savedFilePath,
PartsPath: partsPath,
Parts: 3,
SHA512: sha512Code,
HashType: oget.SHA512,
Hash: sha512Code,
})
return err
}
Expand Down