You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
66 lines
1.5 KiB
66 lines
1.5 KiB
package dao |
|
|
|
import ( |
|
"context" |
|
"io/ioutil" |
|
"net/http" |
|
"net/url" |
|
"time" |
|
|
|
"github.com/json-iterator/go" |
|
|
|
"go-common/app/job/bbq/recall/internal/conf" |
|
"go-common/app/job/bbq/recall/internal/model" |
|
) |
|
|
|
func (d *Dao) queryHDFS(c context.Context, api string, key *conf.BerserkerKey, suffix string) (result *[]byte, err error) { |
|
dt := time.Now().Format("2006-01-02 15:04:05") |
|
sign := d.berserkerSign(key.AppKey, key.Secret, dt, "1.0") |
|
|
|
params := &url.Values{} |
|
params.Set("appKey", key.AppKey) |
|
params.Set("timestamp", dt) |
|
params.Set("version", "1.0") |
|
params.Set("signMethod", "md5") |
|
params.Set("sign", sign) |
|
|
|
fileSuffix := struct { |
|
FileSuffix string `json:"fileSuffix"` |
|
}{ |
|
FileSuffix: suffix, |
|
} |
|
j, err := jsoniter.Marshal(fileSuffix) |
|
params.Set("query", string(j)) |
|
|
|
for retry := 0; retry < 3; retry++ { |
|
resp, err := http.DefaultClient.Get(api + "?" + params.Encode()) |
|
if err != nil { |
|
continue |
|
} |
|
|
|
b, err := ioutil.ReadAll(resp.Body) |
|
resp.Body.Close() |
|
if err == nil && len(b) > 0 { |
|
result = &b |
|
break |
|
} |
|
// sleep 5s berserker limit |
|
time.Sleep(5 * time.Second) |
|
} |
|
|
|
return |
|
} |
|
|
|
func (d *Dao) scanHDFSPath(c context.Context, api string, key *conf.BerserkerKey, suffix string) (result *model.HDFSResult, err error) { |
|
b, err := d.queryHDFS(c, api, key, suffix) |
|
if err != nil { |
|
return |
|
} |
|
result = &model.HDFSResult{} |
|
err = jsoniter.Unmarshal(*b, result) |
|
return |
|
} |
|
|
|
func (d *Dao) loadHDFSFile(c context.Context, api string, key *conf.BerserkerKey, suffix string) (result *[]byte, err error) { |
|
return d.queryHDFS(c, api, key, suffix) |
|
}
|
|
|