mirror of
https://gitee.com/openharmony/developtools_integration_verification
synced 2024-11-23 08:10:12 +00:00
refactor pkg to support more devices
Signed-off-by: huangshan <huangshan9@huawei.com> Change-Id: Ibe479d5840a10130a98ad2f3cd82bf888a95215e
This commit is contained in:
huangshan
parent
f47254e635
commit
4354fa87fd
@ -19,6 +19,7 @@ import (
|
||||
"context"
|
||||
"fotff/pkg"
|
||||
"fotff/pkg/dayu200"
|
||||
"fotff/pkg/gitee_build"
|
||||
"fotff/pkg/mock"
|
||||
"fotff/rec"
|
||||
"fotff/res"
|
||||
@ -26,6 +27,7 @@ import (
|
||||
"fotff/tester/common"
|
||||
"fotff/tester/manual"
|
||||
testermock "fotff/tester/mock"
|
||||
"fotff/tester/pkg_available"
|
||||
"fotff/tester/smoke"
|
||||
"fotff/tester/xdevice"
|
||||
"fotff/utils"
|
||||
@ -37,16 +39,18 @@ import (
|
||||
)
|
||||
|
||||
var newPkgMgrFuncs = map[string]pkg.NewFunc{
|
||||
"mock": mock.NewManager,
|
||||
"dayu200": dayu200.NewManager,
|
||||
"mock": mock.NewManager,
|
||||
"dayu200": dayu200.NewManager,
|
||||
"gitee_build": gitee_build.NewManager,
|
||||
}
|
||||
|
||||
var newTesterFuncs = map[string]tester.NewFunc{
|
||||
"mock": testermock.NewTester,
|
||||
"manual": manual.NewTester,
|
||||
"common": common.NewTester,
|
||||
"xdevice": xdevice.NewTester,
|
||||
"smoke": smoke.NewTester,
|
||||
"mock": testermock.NewTester,
|
||||
"manual": manual.NewTester,
|
||||
"common": common.NewTester,
|
||||
"xdevice": xdevice.NewTester,
|
||||
"smoke": smoke.NewTester,
|
||||
"pkg_available": pkg_available.NewTester,
|
||||
}
|
||||
|
||||
var rootCmd *cobra.Command
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 Huawei Device Co., Ltd.
|
||||
* Copyright (c) 2022-2023 Huawei Device Co., Ltd.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
@ -16,38 +16,62 @@
|
||||
package dayu200
|
||||
|
||||
import (
|
||||
"code.cloudfoundry.org/archiver/extractor"
|
||||
"context"
|
||||
"fmt"
|
||||
"fotff/pkg"
|
||||
"fotff/pkg/gitee_common"
|
||||
"fotff/res"
|
||||
"fotff/utils"
|
||||
"github.com/sirupsen/logrus"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Manager struct {
|
||||
ArchiveDir string `key:"archive_dir" default:"."`
|
||||
FromCI string `key:"download_from_ci" default:"false"`
|
||||
Workspace string `key:"workspace" default:"."`
|
||||
ArchiveDir string `key:"archive_dir" default:"archive"`
|
||||
WatchCI string `key:"watch_ci" default:"false"`
|
||||
Workspace string `key:"workspace" default:"workspace"`
|
||||
Branch string `key:"branch" default:"master"`
|
||||
FlashTool string `key:"flash_tool" default:"python"`
|
||||
LocationIDList string `key:"location_id_list"`
|
||||
locations map[string]string
|
||||
fromCI bool
|
||||
|
||||
*gitee_common.Manager
|
||||
locations map[string]string
|
||||
}
|
||||
|
||||
// These commands are copied from ci project.
|
||||
const (
|
||||
preCompileCMD = `rm -rf prebuilts/clang/ohos/darwin-x86_64/clang-480513;rm -rf prebuilts/clang/ohos/windows-x86_64/clang-480513;rm -rf prebuilts/clang/ohos/linux-x86_64/clang-480513;bash build/prebuilts_download.sh`
|
||||
// compileCMD is copied from ci project and trim useless build-target 'make_test' to enhance build efficiency.
|
||||
compileCMD = `echo 'start' && export NO_DEVTOOL=1 && export CCACHE_LOG_SUFFIX="dayu200-arm32" && export CCACHE_NOHASHDIR="true" && export CCACHE_SLOPPINESS="include_file_ctime" && ./build.sh --product-name rk3568 --ccache --build-target make_all --gn-args enable_notice_collection=false`
|
||||
)
|
||||
|
||||
// This list is copied from ci project. Some of them are not available, has been annotated.
|
||||
var imgList = []string{
|
||||
"out/rk3568/packages/phone/images/MiniLoaderAll.bin",
|
||||
"out/rk3568/packages/phone/images/boot_linux.img",
|
||||
"out/rk3568/packages/phone/images/parameter.txt",
|
||||
"out/rk3568/packages/phone/images/system.img",
|
||||
"out/rk3568/packages/phone/images/uboot.img",
|
||||
"out/rk3568/packages/phone/images/userdata.img",
|
||||
"out/rk3568/packages/phone/images/vendor.img",
|
||||
"out/rk3568/packages/phone/images/resource.img",
|
||||
"out/rk3568/packages/phone/images/config.cfg",
|
||||
"out/rk3568/packages/phone/images/ramdisk.img",
|
||||
// "out/rk3568/packages/phone/images/chipset.img",
|
||||
"out/rk3568/packages/phone/images/sys_prod.img",
|
||||
"out/rk3568/packages/phone/images/chip_prod.img",
|
||||
"out/rk3568/packages/phone/images/updater.img",
|
||||
// "out/rk3568/packages/phone/updater/bin/updater_binary",
|
||||
}
|
||||
|
||||
func NewManager() pkg.Manager {
|
||||
var ret Manager
|
||||
utils.ParseFromConfigFile("dayu200", &ret)
|
||||
var err error
|
||||
if ret.fromCI, err = strconv.ParseBool(ret.FromCI); err != nil {
|
||||
logrus.Panicf("can not parse 'download_from_ci', please check")
|
||||
watchCI, err := strconv.ParseBool(ret.WatchCI)
|
||||
if err != nil {
|
||||
logrus.Panicf("can not parse 'watch_ci', please check")
|
||||
}
|
||||
ret.Manager = gitee_common.NewManager("dayu200", ret.Branch, ret.ArchiveDir, ret.Workspace, watchCI)
|
||||
devs := res.DeviceList()
|
||||
locs := strings.Split(ret.LocationIDList, ",")
|
||||
if len(devs) != len(locs) {
|
||||
@ -57,125 +81,23 @@ func NewManager() pkg.Manager {
|
||||
for i, loc := range locs {
|
||||
ret.locations[devs[i]] = loc
|
||||
}
|
||||
go ret.cleanupOutdated()
|
||||
return &ret
|
||||
}
|
||||
|
||||
func (m *Manager) cleanupOutdated() {
|
||||
t := time.NewTicker(24 * time.Hour)
|
||||
for {
|
||||
<-t.C
|
||||
es, err := os.ReadDir(m.Workspace)
|
||||
if err != nil {
|
||||
logrus.Errorf("can not read %s: %v", m.Workspace, err)
|
||||
continue
|
||||
}
|
||||
for _, e := range es {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(m.Workspace, e.Name())
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
logrus.Errorf("can not read %s info: %v", path, err)
|
||||
continue
|
||||
}
|
||||
if time.Now().Sub(info.ModTime()) > 7*24*time.Hour {
|
||||
logrus.Warnf("%s outdated, cleanning up its contents...", path)
|
||||
m.cleanupPkgFiles(path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) cleanupPkgFiles(path string) {
|
||||
es, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
logrus.Errorf("can not read %s: %v", path, err)
|
||||
return
|
||||
}
|
||||
for _, e := range es {
|
||||
if e.Name() == "manifest_tag.xml" || e.Name() == "__last_issue__" {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(path, e.Name())); err != nil {
|
||||
logrus.Errorf("remove %s fail: %v", filepath.Join(path, e.Name()), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flash function implements pkg.Manager. Flash images in the 'pkg' directory to the given device.
|
||||
// If not all necessary images are available in the 'pkg' directory, will build them.
|
||||
func (m *Manager) Flash(device string, pkg string, ctx context.Context) error {
|
||||
logrus.Infof("now flash %s", pkg)
|
||||
if !m.pkgAvailable(pkg) {
|
||||
logrus.Infof("%s is not available", pkg)
|
||||
if err := m.build(pkg, false, ctx); err != nil {
|
||||
logrus.Errorf("build pkg %s err: %v", pkg, err)
|
||||
logrus.Infof("build pkg %s again...", pkg)
|
||||
if err = m.build(pkg, true, ctx); err != nil {
|
||||
logrus.Errorf("build pkg %s err: %v", pkg, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
buildConfig := gitee_common.BuildConfig{
|
||||
Pkg: pkg,
|
||||
PreCompileCMD: preCompileCMD,
|
||||
CompileCMD: compileCMD,
|
||||
ImgList: imgList,
|
||||
}
|
||||
if err := m.Build(buildConfig, ctx); err != nil {
|
||||
logrus.Errorf("build %s fail, err: %v", pkg, err)
|
||||
return err
|
||||
}
|
||||
logrus.Infof("%s is available now, start to flash it", pkg)
|
||||
return m.flashDevice(device, pkg, ctx)
|
||||
}
|
||||
|
||||
func (m *Manager) Steps(from, to string) (pkgs []string, err error) {
|
||||
if from == to {
|
||||
return nil, fmt.Errorf("steps err: 'from' %s and 'to' %s are the same", from, to)
|
||||
}
|
||||
if c, found := utils.CacheGet("dayu200_steps", from+"__to__"+to); found {
|
||||
logrus.Infof("steps from %s to %s are cached", from, to)
|
||||
logrus.Infof("steps: %v", c.([]string))
|
||||
return c.([]string), nil
|
||||
}
|
||||
if pkgs, err = m.stepsFromGitee(from, to); err != nil {
|
||||
logrus.Errorf("failed to gen steps from gitee, err: %v", err)
|
||||
logrus.Warnf("fallback to getting steps from CI...")
|
||||
if pkgs, err = m.stepsFromCI(from, to); err != nil {
|
||||
return pkgs, err
|
||||
}
|
||||
return pkgs, nil
|
||||
}
|
||||
utils.CacheSet("dayu200_steps", from+"__to__"+to, pkgs)
|
||||
return pkgs, nil
|
||||
}
|
||||
|
||||
func (m *Manager) LastIssue(pkg string) (string, error) {
|
||||
data, err := os.ReadFile(filepath.Join(m.Workspace, pkg, "__last_issue__"))
|
||||
return string(data), err
|
||||
}
|
||||
|
||||
func (m *Manager) GetNewer(cur string) (string, error) {
|
||||
var newFile string
|
||||
if m.fromCI {
|
||||
newFile = m.getNewerFromCI(cur + ".tar.gz")
|
||||
} else {
|
||||
newFile = pkg.GetNewerFileFromDir(m.ArchiveDir, cur+".tar.gz", func(files []os.DirEntry, i, j int) bool {
|
||||
ti, _ := getPackageTime(files[i].Name())
|
||||
tj, _ := getPackageTime(files[j].Name())
|
||||
return ti.Before(tj)
|
||||
})
|
||||
}
|
||||
ex := extractor.NewTgz()
|
||||
dirName := newFile
|
||||
for filepath.Ext(dirName) != "" {
|
||||
dirName = strings.TrimSuffix(dirName, filepath.Ext(dirName))
|
||||
}
|
||||
dir := filepath.Join(m.Workspace, dirName)
|
||||
if _, err := os.Stat(dir); err == nil {
|
||||
return dirName, nil
|
||||
}
|
||||
logrus.Infof("extracting %s to %s...", filepath.Join(m.ArchiveDir, newFile), dir)
|
||||
if err := ex.Extract(filepath.Join(m.ArchiveDir, newFile), dir); err != nil {
|
||||
return dirName, err
|
||||
}
|
||||
return dirName, nil
|
||||
}
|
||||
|
||||
func (m *Manager) PkgDir(pkg string) string {
|
||||
return filepath.Join(m.Workspace, pkg)
|
||||
}
|
||||
|
||||
74
tools/fotff/pkg/gitee_build/gitee_build.go
Normal file
74
tools/fotff/pkg/gitee_build/gitee_build.go
Normal file
@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Copyright (c) 2023 Huawei Device Co., Ltd.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package gitee_build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fotff/pkg"
|
||||
"fotff/pkg/gitee_common"
|
||||
"fotff/utils"
|
||||
"github.com/sirupsen/logrus"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Manager struct {
|
||||
ArchiveDir string `key:"archive_dir" default:"archive"`
|
||||
Workspace string `key:"workspace" default:"workspace"`
|
||||
Branch string `key:"branch" default:"master"`
|
||||
Component string `key:"component"`
|
||||
PreCompileCMD string `key:"pre_compile_cmd"`
|
||||
CompileCMD string `key:"compile_cmd"`
|
||||
ImageList []string `key:"image_list"`
|
||||
|
||||
*gitee_common.Manager
|
||||
}
|
||||
|
||||
func NewManager() pkg.Manager {
|
||||
var ret Manager
|
||||
utils.ParseFromConfigFile("gitee_build", &ret)
|
||||
ret.Manager = gitee_common.NewManager(ret.Component, ret.Branch, ret.ArchiveDir, ret.Workspace, true)
|
||||
return &ret
|
||||
}
|
||||
|
||||
func (m *Manager) GetNewer(cur string) (string, error) {
|
||||
return m.GetNewerOrFail(cur)
|
||||
}
|
||||
|
||||
// Flash function implements pkg.Manager. Since this gitee_build just tests package buildings,
|
||||
// there is no need to flash images actually, just build it and return nil unconditionally.
|
||||
func (m *Manager) Flash(device string, pkg string, ctx context.Context) error {
|
||||
logrus.Infof("now flash %s", pkg)
|
||||
buildConfig := gitee_common.BuildConfig{
|
||||
Pkg: pkg,
|
||||
PreCompileCMD: m.PreCompileCMD,
|
||||
CompileCMD: m.CompileCMD,
|
||||
ImgList: m.ImageList,
|
||||
}
|
||||
if m.PkgAvailable(buildConfig) {
|
||||
return nil
|
||||
}
|
||||
if strings.Contains(buildConfig.Pkg, "build_fail") {
|
||||
logrus.Warnf("here is a known build_fail token package")
|
||||
} else {
|
||||
if err := m.BuildNoRetry(buildConfig, true, ctx); err != nil {
|
||||
logrus.Warnf("build %s fail, err: %v", pkg, err)
|
||||
} else {
|
||||
logrus.Infof("%s is available now", pkg)
|
||||
}
|
||||
}
|
||||
logrus.Infof("since fotff just tests package buildings, there is no need to flash images actually, mark flash as a success unconditionally")
|
||||
return nil
|
||||
}
|
||||
@ -13,7 +13,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package dayu200
|
||||
package gitee_common
|
||||
|
||||
import (
|
||||
"context"
|
||||
@ -25,48 +25,47 @@ import (
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// These commands are copied from ci project.
|
||||
const (
|
||||
preCompileCMD = `rm -rf prebuilts/clang/ohos/darwin-x86_64/clang-480513;rm -rf prebuilts/clang/ohos/windows-x86_64/clang-480513;rm -rf prebuilts/clang/ohos/linux-x86_64/clang-480513;bash build/prebuilts_download.sh`
|
||||
// compileCMD is copied from ci project and trim useless build-target 'make_test' to enhance build efficiency.
|
||||
compileCMD = `echo 'start' && export NO_DEVTOOL=1 && export CCACHE_LOG_SUFFIX="dayu200-arm32" && export CCACHE_NOHASHDIR="true" && export CCACHE_SLOPPINESS="include_file_ctime" && ./build.sh --product-name rk3568 --ccache --build-target make_all --gn-args enable_notice_collection=false`
|
||||
rmOutCMD = `rm -rf out`
|
||||
)
|
||||
|
||||
// This list is copied from ci project. Some of them are not available, has been annotated.
|
||||
var imgList = []string{
|
||||
"out/rk3568/packages/phone/images/MiniLoaderAll.bin",
|
||||
"out/rk3568/packages/phone/images/boot_linux.img",
|
||||
"out/rk3568/packages/phone/images/parameter.txt",
|
||||
"out/rk3568/packages/phone/images/system.img",
|
||||
"out/rk3568/packages/phone/images/uboot.img",
|
||||
"out/rk3568/packages/phone/images/userdata.img",
|
||||
"out/rk3568/packages/phone/images/vendor.img",
|
||||
"out/rk3568/packages/phone/images/resource.img",
|
||||
"out/rk3568/packages/phone/images/config.cfg",
|
||||
"out/rk3568/packages/phone/images/ramdisk.img",
|
||||
// "out/rk3568/packages/phone/images/chipset.img",
|
||||
"out/rk3568/packages/phone/images/sys_prod.img",
|
||||
"out/rk3568/packages/phone/images/chip_prod.img",
|
||||
"out/rk3568/packages/phone/images/updater.img",
|
||||
// "out/rk3568/packages/phone/updater/bin/updater_binary",
|
||||
type BuildConfig struct {
|
||||
Pkg string
|
||||
PreCompileCMD string
|
||||
CompileCMD string
|
||||
ImgList []string
|
||||
}
|
||||
|
||||
// pkgAvailable returns true if all necessary images are all available to flash.
|
||||
func (m *Manager) pkgAvailable(pkg string) bool {
|
||||
for _, img := range imgList {
|
||||
func (m *Manager) Build(config BuildConfig, ctx context.Context) error {
|
||||
if m.PkgAvailable(config) {
|
||||
return nil
|
||||
}
|
||||
logrus.Infof("%s is not available", config.Pkg)
|
||||
err := m.BuildNoRetry(config, false, ctx)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
logrus.Errorf("build pkg %s err: %v", config.Pkg, err)
|
||||
logrus.Infof("rm out and build pkg %s again...", config.Pkg)
|
||||
err = m.BuildNoRetry(config, true, ctx)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
logrus.Errorf("build pkg %s err: %v", config.Pkg, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// PkgAvailable returns true if all necessary images are all available to flash.
|
||||
func (m *Manager) PkgAvailable(config BuildConfig) bool {
|
||||
for _, img := range config.ImgList {
|
||||
imgName := filepath.Base(img)
|
||||
if _, err := os.Stat(filepath.Join(m.Workspace, pkg, imgName)); err != nil {
|
||||
if _, err := os.Stat(filepath.Join(m.Workspace, config.Pkg, imgName)); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// build obtain an available server, download corresponding codes, and run compile commands
|
||||
// BuildNoRetry obtain an available server, download corresponding codes, and run compile commands
|
||||
// to build the corresponding package images, then transfer these images to the 'pkg' directory.
|
||||
func (m *Manager) build(pkg string, rm bool, ctx context.Context) error {
|
||||
logrus.Infof("now build %s", pkg)
|
||||
func (m *Manager) BuildNoRetry(config BuildConfig, rm bool, ctx context.Context) error {
|
||||
logrus.Infof("now Build %s", config.Pkg)
|
||||
server := res.GetBuildServer()
|
||||
defer res.ReleaseBuildServer(server)
|
||||
cmd := fmt.Sprintf("mkdir -p %s && cd %s && repo init -u https://gitee.com/openharmony/manifest.git", server.WorkSpace, server.WorkSpace)
|
||||
@ -74,7 +73,7 @@ func (m *Manager) build(pkg string, rm bool, ctx context.Context) error {
|
||||
return fmt.Errorf("remote: mkdir error: %w", err)
|
||||
}
|
||||
if err := utils.TransFileViaSSH(utils.Upload, server.Addr, server.User, server.Passwd,
|
||||
fmt.Sprintf("%s/.repo/manifest.xml", server.WorkSpace), filepath.Join(m.Workspace, pkg, "manifest_tag.xml")); err != nil {
|
||||
fmt.Sprintf("%s/.repo/manifest.xml", server.WorkSpace), filepath.Join(m.Workspace, config.Pkg, "manifest_tag.xml")); err != nil {
|
||||
return fmt.Errorf("upload and replace manifest error: %w", err)
|
||||
}
|
||||
// 'git lfs install' may fail due to some git hooks. Call 'git lfs update --force' before install to avoid this situation.
|
||||
@ -82,25 +81,25 @@ func (m *Manager) build(pkg string, rm bool, ctx context.Context) error {
|
||||
if err := utils.RunCmdViaSSHContext(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil {
|
||||
return fmt.Errorf("remote: repo sync error: %w", err)
|
||||
}
|
||||
cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, preCompileCMD)
|
||||
cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, config.PreCompileCMD)
|
||||
if err := utils.RunCmdViaSSHContextNoRetry(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil {
|
||||
return fmt.Errorf("remote: pre-compile command error: %w", err)
|
||||
}
|
||||
if rm {
|
||||
cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, rmOutCMD)
|
||||
cmd = fmt.Sprintf("cd %s && rm -rf out", server.WorkSpace)
|
||||
if err := utils.RunCmdViaSSHContext(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil {
|
||||
return fmt.Errorf("remote: rm ./out command error: %w", err)
|
||||
}
|
||||
}
|
||||
cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, compileCMD)
|
||||
cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, config.CompileCMD)
|
||||
if err := utils.RunCmdViaSSHContextNoRetry(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil {
|
||||
return fmt.Errorf("remote: compile command error: %w", err)
|
||||
}
|
||||
// has been built already, pitiful if canceled, so continue copying
|
||||
for _, f := range imgList {
|
||||
for _, f := range config.ImgList {
|
||||
imgName := filepath.Base(f)
|
||||
if err := utils.TransFileViaSSH(utils.Download, server.Addr, server.User, server.Passwd,
|
||||
fmt.Sprintf("%s/%s", server.WorkSpace, f), filepath.Join(m.Workspace, pkg, imgName)); err != nil {
|
||||
fmt.Sprintf("%s/%s", server.WorkSpace, f), filepath.Join(m.Workspace, config.Pkg, imgName)); err != nil {
|
||||
return fmt.Errorf("download file %s error: %w", f, err)
|
||||
}
|
||||
}
|
||||
151
tools/fotff/pkg/gitee_common/common.go
Normal file
151
tools/fotff/pkg/gitee_common/common.go
Normal file
@ -0,0 +1,151 @@
|
||||
/*
|
||||
* Copyright (c) 2022 Huawei Device Co., Ltd.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package gitee_common
|
||||
|
||||
import (
|
||||
"code.cloudfoundry.org/archiver/extractor"
|
||||
"context"
|
||||
"fmt"
|
||||
"fotff/utils"
|
||||
"github.com/sirupsen/logrus"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Manager struct {
|
||||
Component string
|
||||
Branch string
|
||||
ArchiveDir string
|
||||
Workspace string
|
||||
WatchCI bool
|
||||
}
|
||||
|
||||
func NewManager(component string, branch string, archiveDir string, workspace string, watchCI bool) *Manager {
|
||||
var ret = Manager{
|
||||
Component: component,
|
||||
Branch: branch,
|
||||
ArchiveDir: archiveDir,
|
||||
Workspace: workspace,
|
||||
WatchCI: watchCI,
|
||||
}
|
||||
go ret.cleanupOutdated()
|
||||
return &ret
|
||||
}
|
||||
|
||||
func (m *Manager) cleanupOutdated() {
|
||||
t := time.NewTicker(24 * time.Hour)
|
||||
for {
|
||||
<-t.C
|
||||
es, err := os.ReadDir(m.Workspace)
|
||||
if err != nil {
|
||||
logrus.Errorf("can not read %s: %v", m.Workspace, err)
|
||||
continue
|
||||
}
|
||||
for _, e := range es {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(m.Workspace, e.Name())
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
logrus.Errorf("can not read %s info: %v", path, err)
|
||||
continue
|
||||
}
|
||||
if time.Now().Sub(info.ModTime()) > 7*24*time.Hour {
|
||||
logrus.Warnf("%s outdated, cleanning up its contents...", path)
|
||||
m.cleanupPkgFiles(path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) cleanupPkgFiles(path string) {
|
||||
es, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
logrus.Errorf("can not read %s: %v", path, err)
|
||||
return
|
||||
}
|
||||
for _, e := range es {
|
||||
if e.Name() == "manifest_tag.xml" || e.Name() == "__last_issue__" {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(path, e.Name())); err != nil {
|
||||
logrus.Errorf("remove %s fail: %v", filepath.Join(path, e.Name()), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flash function implements pkg.Manager. Flash images in the 'pkg' directory to the given device.
|
||||
func (m *Manager) Flash(device string, pkg string, ctx context.Context) error {
|
||||
logrus.Warnf("not implemented yet")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) Steps(from, to string) (pkgs []string, err error) {
|
||||
if from == to {
|
||||
return nil, fmt.Errorf("steps err: 'from' %s and 'to' %s are the same", from, to)
|
||||
}
|
||||
if c, found := utils.CacheGet(fmt.Sprintf("%s_steps", m.Component), from+"__to__"+to); found {
|
||||
logrus.Infof("steps from %s to %s are cached", from, to)
|
||||
logrus.Infof("steps: %v", c.([]string))
|
||||
return c.([]string), nil
|
||||
}
|
||||
if pkgs, err = m.stepsFromGitee(from, to); err != nil {
|
||||
logrus.Errorf("failed to gen steps from gitee, err: %v", err)
|
||||
logrus.Warnf("fallback to getting steps from CI...")
|
||||
if pkgs, err = m.stepsFromCI(from, to); err != nil {
|
||||
return pkgs, err
|
||||
}
|
||||
return pkgs, nil
|
||||
}
|
||||
utils.CacheSet(fmt.Sprintf("%s_steps", m.Component), from+"__to__"+to, pkgs)
|
||||
return pkgs, nil
|
||||
}
|
||||
|
||||
func (m *Manager) LastIssue(pkg string) (string, error) {
|
||||
data, err := os.ReadFile(filepath.Join(m.Workspace, pkg, "__last_issue__"))
|
||||
return string(data), err
|
||||
}
|
||||
|
||||
func (m *Manager) GetNewer(cur string) (string, error) {
|
||||
var newFile string
|
||||
if m.WatchCI {
|
||||
newFile = m.getNewerFromCI(cur + ".tar.gz")
|
||||
} else {
|
||||
newFile = m.getNewerFileFromDir(cur+".tar.gz", func(files []os.DirEntry, i, j int) bool {
|
||||
ti, _ := parseTime(files[i].Name())
|
||||
tj, _ := parseTime(files[j].Name())
|
||||
return ti.Before(tj)
|
||||
})
|
||||
}
|
||||
ex := extractor.NewTgz()
|
||||
dirName := strings.TrimSuffix(newFile, ".tar.gz")
|
||||
dir := filepath.Join(m.Workspace, dirName)
|
||||
if _, err := os.Stat(dir); err == nil {
|
||||
return dirName, nil
|
||||
}
|
||||
logrus.Infof("extracting %s to %s...", filepath.Join(m.ArchiveDir, newFile), dir)
|
||||
if err := ex.Extract(filepath.Join(m.ArchiveDir, newFile), dir); err != nil {
|
||||
return dirName, err
|
||||
}
|
||||
return dirName, nil
|
||||
}
|
||||
|
||||
func (m *Manager) PkgDir(pkg string) string {
|
||||
return filepath.Join(m.Workspace, pkg)
|
||||
}
|
||||
@ -13,10 +13,11 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package dayu200
|
||||
package gitee_common
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"fotff/utils"
|
||||
"github.com/sirupsen/logrus"
|
||||
"io"
|
||||
@ -45,22 +46,18 @@ type DailyBuildsResp struct {
|
||||
}
|
||||
|
||||
type DailyBuild struct {
|
||||
Id string `json:"id"`
|
||||
ImgObsPath string `json:"imgObsPath"`
|
||||
CurrentStatus string `json:"currentStatus"`
|
||||
BuildStartTime string `json:"buildStartTime"`
|
||||
BuildFailReason string `json:"buildFailReason"`
|
||||
Id string `json:"id"`
|
||||
ObsPath string `json:"obsPath"`
|
||||
ImgObsPath string `json:"imgObsPath"`
|
||||
}
|
||||
|
||||
func (m *Manager) getNewerFromCI(cur string) string {
|
||||
func (m *Manager) loopCI(param DailyBuildsQueryParam, cur string, getFn func(cur string, resp *DailyBuild) string) string {
|
||||
for {
|
||||
file := func() string {
|
||||
var q = DailyBuildsQueryParam{
|
||||
ProjectName: "openharmony",
|
||||
Branch: m.Branch,
|
||||
Component: "dayu200",
|
||||
BuildStatus: "success",
|
||||
PageNum: 1,
|
||||
PageSize: 1,
|
||||
}
|
||||
data, err := json.Marshal(q)
|
||||
data, err := json.Marshal(param)
|
||||
if err != nil {
|
||||
logrus.Errorf("can not marshal query param: %v", err)
|
||||
return ""
|
||||
@ -75,19 +72,13 @@ func (m *Manager) getNewerFromCI(cur string) string {
|
||||
logrus.Errorf("can not unmarshal resp [%s]: %v", string(resp), err)
|
||||
return ""
|
||||
}
|
||||
if len(dailyBuildsResp.Result.DailyBuildVos) != 0 {
|
||||
url := dailyBuildsResp.Result.DailyBuildVos[0].ImgObsPath
|
||||
if filepath.Base(url) != cur {
|
||||
logrus.Infof("new package found, name: %s", filepath.Base(url))
|
||||
file, err := m.downloadToWorkspace(url)
|
||||
if err != nil {
|
||||
logrus.Errorf("can not download package %s: %v", url, err)
|
||||
return ""
|
||||
}
|
||||
return file
|
||||
}
|
||||
if len(dailyBuildsResp.Result.DailyBuildVos) == 0 {
|
||||
return ""
|
||||
}
|
||||
return ""
|
||||
if dailyBuildsResp.Result.DailyBuildVos[0].CurrentStatus != "end" {
|
||||
return ""
|
||||
}
|
||||
return getFn(cur, dailyBuildsResp.Result.DailyBuildVos[0])
|
||||
}()
|
||||
if file != "" {
|
||||
return file
|
||||
@ -96,7 +87,39 @@ func (m *Manager) getNewerFromCI(cur string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) getNewerFromCI(cur string) string {
|
||||
return m.loopCI(DailyBuildsQueryParam{
|
||||
ProjectName: "openharmony",
|
||||
Branch: m.Branch,
|
||||
Component: m.Component,
|
||||
BuildStatus: "success",
|
||||
PageNum: 1,
|
||||
PageSize: 1,
|
||||
}, cur, m.getNewerDailyBuild)
|
||||
}
|
||||
|
||||
func (m *Manager) getNewerDailyBuild(cur string, db *DailyBuild) string {
|
||||
p := db.ImgObsPath
|
||||
if p == "" {
|
||||
p = db.ObsPath
|
||||
}
|
||||
if filepath.Base(p) == cur {
|
||||
return ""
|
||||
}
|
||||
logrus.Infof("new package found, name: %s", filepath.Base(p))
|
||||
file, err := m.downloadToWorkspace(p)
|
||||
if err != nil {
|
||||
logrus.Errorf("can not download package %s: %v", p, err)
|
||||
return ""
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
func (m *Manager) downloadToWorkspace(url string) (string, error) {
|
||||
if _, err := parseTime(filepath.Base(url)); err != nil {
|
||||
logrus.Errorf("can not get package time from %s, skipping", filepath.Base(url))
|
||||
return "", fmt.Errorf("can not get package time from %s, skipping", filepath.Base(url))
|
||||
}
|
||||
logrus.Infof("downloading %s", url)
|
||||
resp, err := utils.DoSimpleHttpReqRaw(http.MethodGet, url, nil, nil)
|
||||
if err != nil {
|
||||
45
tools/fotff/pkg/gitee_common/get_newer_dir.go
Normal file
45
tools/fotff/pkg/gitee_common/get_newer_dir.go
Normal file
@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Copyright (c) 2023 Huawei Device Co., Ltd.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package gitee_common
|
||||
|
||||
import (
|
||||
"github.com/sirupsen/logrus"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (m *Manager) getNewerFileFromDir(cur string, less func(files []os.DirEntry, i, j int) bool) string {
|
||||
for {
|
||||
files, err := os.ReadDir(m.ArchiveDir)
|
||||
if err != nil {
|
||||
logrus.Errorf("read dir %s err: %s", m.ArchiveDir, err)
|
||||
time.Sleep(10 * time.Second)
|
||||
continue
|
||||
}
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return less(files, i, j)
|
||||
})
|
||||
if len(files) != 0 {
|
||||
f := files[len(files)-1]
|
||||
if f.Name() != cur {
|
||||
logrus.Infof("new package found, name: %s", f.Name())
|
||||
return f.Name()
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
}
|
||||
111
tools/fotff/pkg/gitee_common/get_newer_or_fail.go
Normal file
111
tools/fotff/pkg/gitee_common/get_newer_or_fail.go
Normal file
@ -0,0 +1,111 @@
|
||||
/*
|
||||
* Copyright (c) 2023 Huawei Device Co., Ltd.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package gitee_common
|
||||
|
||||
import (
|
||||
"code.cloudfoundry.org/archiver/compressor"
|
||||
"code.cloudfoundry.org/archiver/extractor"
|
||||
"fmt"
|
||||
"fotff/utils"
|
||||
"github.com/sirupsen/logrus"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (m *Manager) GetNewerOrFail(cur string) (string, error) {
|
||||
newFile := m.getNewerOrFailFromCI(cur + ".tar.gz")
|
||||
ex := extractor.NewTgz()
|
||||
dirName := strings.TrimSuffix(newFile, ".tar.gz")
|
||||
dir := filepath.Join(m.Workspace, dirName)
|
||||
if _, err := os.Stat(dir); err == nil {
|
||||
return dirName, nil
|
||||
}
|
||||
logrus.Infof("extracting %s to %s...", filepath.Join(m.ArchiveDir, newFile), dir)
|
||||
if err := ex.Extract(filepath.Join(m.ArchiveDir, newFile), dir); err != nil {
|
||||
return dirName, err
|
||||
}
|
||||
return dirName, nil
|
||||
}
|
||||
|
||||
func (m *Manager) getNewerOrFailFromCI(cur string) string {
|
||||
return m.loopCI(DailyBuildsQueryParam{
|
||||
ProjectName: "openharmony",
|
||||
Branch: m.Branch,
|
||||
Component: m.Component,
|
||||
PageNum: 1,
|
||||
PageSize: 1,
|
||||
}, cur, m.getNewerDailyBuildOrFail)
|
||||
}
|
||||
|
||||
func (m *Manager) getNewerDailyBuildOrFail(cur string, db *DailyBuild) string {
|
||||
switch db.BuildFailReason {
|
||||
case "":
|
||||
return m.getNewerDailyBuild(cur, db)
|
||||
case "compile_failed":
|
||||
lastSuccessTime, err := parseTime(cur)
|
||||
if err != nil {
|
||||
logrus.Errorf("can not get package time from %s, skipping", cur)
|
||||
return ""
|
||||
}
|
||||
nowFailTime, err := parseTime(db.BuildStartTime)
|
||||
if err != nil {
|
||||
logrus.Errorf("can not get time from %s, skipping", cur)
|
||||
return ""
|
||||
}
|
||||
if lastSuccessTime == nowFailTime {
|
||||
return ""
|
||||
}
|
||||
return m.genFailedPackage(lastSuccessTime, nowFailTime)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) genFailedPackage(lastSuccessTime, failedBuildStartTime time.Time) string {
|
||||
pkg := fmt.Sprintf("%s_%s_build_fail", m.Component, failedBuildStartTime.Format("20060102_150405"))
|
||||
logrus.Infof("getting failed package manifest for %s(%s) at %s", m.Component, m.Branch, failedBuildStartTime)
|
||||
tags, err := m.getAllTags(lastSuccessTime, failedBuildStartTime)
|
||||
if err != nil {
|
||||
logrus.Errorf("can not get latest tag from ci, err: %v", err)
|
||||
return ""
|
||||
}
|
||||
if len(tags) == 0 {
|
||||
logrus.Error("can not get latest tag from ci, tag list is empty")
|
||||
return ""
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(m.Workspace, pkg), 0750); err != nil {
|
||||
logrus.Errorf("can not mkdir %s, err: %v", pkg, err)
|
||||
return ""
|
||||
}
|
||||
resp, err := utils.DoSimpleHttpReq(http.MethodGet, tags[len(tags)-1].TagFileURL, nil, nil)
|
||||
if err != nil {
|
||||
logrus.Errorf("can not get latest tag file from ci, err: %v", err)
|
||||
return ""
|
||||
}
|
||||
err = os.WriteFile(filepath.Join(m.Workspace, pkg, "manifest_tag.xml"), resp, 0640)
|
||||
if err != nil {
|
||||
logrus.Errorf("can not write latest tag file, err: %v", err)
|
||||
return ""
|
||||
}
|
||||
if err := compressor.NewTgz().Compress(filepath.Join(m.Workspace, pkg), filepath.Join(m.ArchiveDir, pkg+".tar.gz")); err != nil {
|
||||
logrus.Errorf("can not write latest tag file, err: %v", err)
|
||||
return ""
|
||||
}
|
||||
return pkg + ".tar.gz"
|
||||
}
|
||||
@ -13,7 +13,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package dayu200
|
||||
package gitee_common
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
@ -53,11 +53,11 @@ type Tag struct {
|
||||
}
|
||||
|
||||
func (m *Manager) stepsFromCI(from, to string) (pkgs []string, err error) {
|
||||
startTime, err := getPackageTime(from)
|
||||
startTime, err := parseTime(from)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
endTime, err := getPackageTime(to)
|
||||
endTime, err := parseTime(to)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -69,9 +69,6 @@ func (m *Manager) getAllStepsFromTags(from, to time.Time) (pkgs []string, err er
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(tags, func(i, j int) bool {
|
||||
return tags[i].Timestamp < tags[j].Timestamp
|
||||
})
|
||||
for _, tag := range tags {
|
||||
pkg, err := m.genTagPackage(tag)
|
||||
if err != nil {
|
||||
@ -125,6 +122,9 @@ func (m *Manager) getAllTags(from, to time.Time) (ret []*Tag, err error) {
|
||||
}
|
||||
pageNum++
|
||||
}
|
||||
sort.Slice(ret, func(i, j int) bool {
|
||||
return ret[i].Timestamp < ret[j].Timestamp
|
||||
})
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
@ -1,365 +1,367 @@
|
||||
/*
|
||||
* Copyright (c) 2022 Huawei Device Co., Ltd.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package dayu200
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"fotff/vcs"
|
||||
"fotff/vcs/gitee"
|
||||
"github.com/huandu/go-clone"
|
||||
"github.com/sirupsen/logrus"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type IssueInfo struct {
|
||||
visited bool
|
||||
RelatedIssues []string
|
||||
MRs []*gitee.Commit
|
||||
StructCTime string
|
||||
StructureUpdates []*vcs.ProjectUpdate
|
||||
}
|
||||
|
||||
type Step struct {
|
||||
IssueURLs []string
|
||||
MRs []*gitee.Commit
|
||||
StructCTime string
|
||||
StructureUpdates []*vcs.ProjectUpdate
|
||||
}
|
||||
|
||||
func (m *Manager) stepsFromGitee(from, to string) (pkgs []string, err error) {
|
||||
updates, err := m.getRepoUpdates(from, to)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
startTime, err := getPackageTime(from)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
endTime, err := getPackageTime(to)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logrus.Infof("find %d repo updates from %s to %s", len(updates), from, to)
|
||||
steps, err := getAllStepsFromGitee(startTime, endTime, m.Branch, updates)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logrus.Infof("find total %d steps from %s to %s", len(steps), from, to)
|
||||
baseManifest, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, from, "manifest_tag.xml"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, step := range steps {
|
||||
var newPkg string
|
||||
if newPkg, baseManifest, err = m.genStepPackage(baseManifest, step); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pkgs = append(pkgs, newPkg)
|
||||
}
|
||||
return pkgs, nil
|
||||
}
|
||||
|
||||
func (m *Manager) getRepoUpdates(from, to string) (updates []vcs.ProjectUpdate, err error) {
|
||||
m1, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, from, "manifest_tag.xml"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m2, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, to, "manifest_tag.xml"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return vcs.GetRepoUpdates(m1, m2)
|
||||
}
|
||||
|
||||
func getAllStepsFromGitee(startTime, endTime time.Time, branch string, updates []vcs.ProjectUpdate) (ret []Step, err error) {
|
||||
allMRs, err := getAllMRs(startTime, endTime, branch, updates)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
issueInfos, err := combineMRsToIssue(allMRs, branch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return combineIssuesToStep(issueInfos)
|
||||
}
|
||||
|
||||
func getAllMRs(startTime, endTime time.Time, branch string, updates []vcs.ProjectUpdate) (allMRs []*gitee.Commit, err error) {
|
||||
var once sync.Once
|
||||
for _, update := range updates {
|
||||
var prs []*gitee.Commit
|
||||
if update.P1.StructureDiff(update.P2) {
|
||||
once.Do(func() {
|
||||
prs, err = gitee.GetBetweenTimeMRs("openharmony", "manifest", branch, startTime, endTime)
|
||||
})
|
||||
if update.P1 != nil {
|
||||
var p1 []*gitee.Commit
|
||||
p1, err = gitee.GetBetweenTimeMRs("openharmony", update.P1.Name, branch, startTime, endTime)
|
||||
prs = append(prs, p1...)
|
||||
}
|
||||
if update.P2 != nil {
|
||||
var p2 []*gitee.Commit
|
||||
p2, err = gitee.GetBetweenTimeMRs("openharmony", update.P2.Name, branch, startTime, endTime)
|
||||
prs = append(prs, p2...)
|
||||
}
|
||||
} else {
|
||||
prs, err = gitee.GetBetweenMRs(gitee.CompareParam{
|
||||
Head: update.P2.Revision,
|
||||
Base: update.P1.Revision,
|
||||
Owner: "openharmony",
|
||||
Repo: update.P2.Name,
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allMRs = append(allMRs, prs...)
|
||||
}
|
||||
logrus.Infof("find total %d merge request commits of all repo updates", len(allMRs))
|
||||
return
|
||||
}
|
||||
|
||||
func combineMRsToIssue(allMRs []*gitee.Commit, branch string) (map[string]*IssueInfo, error) {
|
||||
ret := make(map[string]*IssueInfo)
|
||||
for _, mr := range allMRs {
|
||||
num, err := strconv.Atoi(strings.Trim(regexp.MustCompile(`!\d+ `).FindString(mr.Commit.Message), "! "))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse MR message for %s fail: %s", mr.URL, err)
|
||||
}
|
||||
issues, err := gitee.GetMRIssueURL(mr.Owner, mr.Repo, num)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(issues) == 0 {
|
||||
issues = []string{mr.URL}
|
||||
}
|
||||
var scs []*vcs.ProjectUpdate
|
||||
var scTime string
|
||||
if mr.Owner == "openharmony" && mr.Repo == "manifest" {
|
||||
if scTime, scs, err = parseStructureUpdates(mr, branch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for i, issue := range issues {
|
||||
if _, ok := ret[issue]; !ok {
|
||||
ret[issue] = &IssueInfo{
|
||||
MRs: []*gitee.Commit{mr},
|
||||
RelatedIssues: append(issues[:i], issues[i+1:]...),
|
||||
StructCTime: scTime,
|
||||
StructureUpdates: scs,
|
||||
}
|
||||
} else {
|
||||
ret[issue] = &IssueInfo{
|
||||
MRs: append(ret[issue].MRs, mr),
|
||||
RelatedIssues: append(ret[issue].RelatedIssues, append(issues[:i], issues[i+1:]...)...),
|
||||
StructCTime: scTime,
|
||||
StructureUpdates: append(ret[issue].StructureUpdates, scs...),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
logrus.Infof("find total %d issues of all repo updates", len(ret))
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func combineOtherRelatedIssue(parent, self *IssueInfo, all map[string]*IssueInfo) {
|
||||
if self.visited {
|
||||
return
|
||||
}
|
||||
self.visited = true
|
||||
for _, other := range self.RelatedIssues {
|
||||
if son, ok := all[other]; ok {
|
||||
combineOtherRelatedIssue(self, son, all)
|
||||
delete(all, other)
|
||||
}
|
||||
}
|
||||
parent.RelatedIssues = deDupIssues(append(parent.RelatedIssues, self.RelatedIssues...))
|
||||
parent.MRs = deDupMRs(append(parent.MRs, self.MRs...))
|
||||
parent.StructureUpdates = deDupProjectUpdates(append(parent.StructureUpdates, self.StructureUpdates...))
|
||||
if len(parent.StructCTime) != 0 && parent.StructCTime < self.StructCTime {
|
||||
parent.StructCTime = self.StructCTime
|
||||
}
|
||||
}
|
||||
|
||||
func deDupProjectUpdates(us []*vcs.ProjectUpdate) (retMRs []*vcs.ProjectUpdate) {
|
||||
dupIndexes := make([]bool, len(us))
|
||||
for i := range us {
|
||||
for j := i + 1; j < len(us); j++ {
|
||||
if us[j].P1 == us[i].P1 && us[j].P2 == us[i].P2 {
|
||||
dupIndexes[j] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, dup := range dupIndexes {
|
||||
if dup {
|
||||
continue
|
||||
}
|
||||
retMRs = append(retMRs, us[i])
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func deDupMRs(mrs []*gitee.Commit) (retMRs []*gitee.Commit) {
|
||||
tmp := make(map[string]*gitee.Commit)
|
||||
for _, m := range mrs {
|
||||
tmp[m.SHA] = m
|
||||
}
|
||||
for _, m := range tmp {
|
||||
retMRs = append(retMRs, m)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func deDupIssues(issues []string) (retIssues []string) {
|
||||
tmp := make(map[string]string)
|
||||
for _, i := range issues {
|
||||
tmp[i] = i
|
||||
}
|
||||
for _, i := range tmp {
|
||||
retIssues = append(retIssues, i)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// parseStructureUpdates get changed XMLs and parse it to recognize repo structure changes.
|
||||
// Since we do not care which revision a repo was, P1 is not welly handled, just assign it not nil for performance.
|
||||
func parseStructureUpdates(commit *gitee.Commit, branch string) (string, []*vcs.ProjectUpdate, error) {
|
||||
tmp := make(map[string]vcs.ProjectUpdate)
|
||||
if len(commit.Files) == 0 {
|
||||
// commit that queried from MR req does not contain file details, should fetch again
|
||||
var err error
|
||||
if commit, err = gitee.GetCommit(commit.Owner, commit.Repo, commit.SHA); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
for _, f := range commit.Files {
|
||||
if filepath.Ext(f.Filename) != ".xml" {
|
||||
continue
|
||||
}
|
||||
if err := parseFilePatch(f.Patch, tmp); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
var ret []*vcs.ProjectUpdate
|
||||
for _, pu := range tmp {
|
||||
projectUpdateCopy := pu
|
||||
ret = append(ret, &projectUpdateCopy)
|
||||
}
|
||||
for _, pu := range ret {
|
||||
if pu.P1 == nil && pu.P2 != nil {
|
||||
lastCommit, err := gitee.GetLatestMRBefore("openharmony", pu.P2.Name, branch, commit.Commit.Committer.Date)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
pu.P2.Revision = lastCommit.SHA
|
||||
}
|
||||
}
|
||||
return commit.Commit.Committer.Date, ret, nil
|
||||
}
|
||||
|
||||
func parseFilePatch(str string, m map[string]vcs.ProjectUpdate) error {
|
||||
sc := bufio.NewScanner(bytes.NewBuffer([]byte(str)))
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
var p vcs.Project
|
||||
if strings.HasPrefix(line, "-") {
|
||||
if err := xml.Unmarshal([]byte(line[1:]), &p); err == nil {
|
||||
m[p.Name] = vcs.ProjectUpdate{P1: &p, P2: m[p.Name].P2}
|
||||
}
|
||||
} else if strings.HasPrefix(line, "+") {
|
||||
if err := xml.Unmarshal([]byte(line[1:]), &p); err == nil {
|
||||
m[p.Name] = vcs.ProjectUpdate{P1: m[p.Name].P1, P2: &p}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func combineIssuesToStep(issueInfos map[string]*IssueInfo) (ret []Step, err error) {
|
||||
for _, info := range issueInfos {
|
||||
combineOtherRelatedIssue(info, info, issueInfos)
|
||||
}
|
||||
for issue, infos := range issueInfos {
|
||||
sort.Slice(infos.MRs, func(i, j int) bool {
|
||||
// move the latest MR to the first place, use its merged_time to represent the update time of the issue
|
||||
return infos.MRs[i].Commit.Committer.Date > infos.MRs[j].Commit.Committer.Date
|
||||
})
|
||||
ret = append(ret, Step{
|
||||
IssueURLs: append(infos.RelatedIssues, issue),
|
||||
MRs: infos.MRs,
|
||||
StructCTime: infos.StructCTime,
|
||||
StructureUpdates: infos.StructureUpdates})
|
||||
}
|
||||
sort.Slice(ret, func(i, j int) bool {
|
||||
ti, tj := ret[i].MRs[0].Commit.Committer.Date, ret[j].MRs[0].Commit.Committer.Date
|
||||
if len(ret[i].StructCTime) != 0 {
|
||||
ti = ret[i].StructCTime
|
||||
}
|
||||
if len(ret[j].StructCTime) != 0 {
|
||||
ti = ret[j].StructCTime
|
||||
}
|
||||
return ti < tj
|
||||
})
|
||||
logrus.Infof("find total %d steps of all issues", len(ret))
|
||||
return
|
||||
}
|
||||
|
||||
var simpleRegTimeInPkgName = regexp.MustCompile(`\d{8}_\d{6}`)
|
||||
|
||||
func getPackageTime(pkg string) (time.Time, error) {
|
||||
return time.ParseInLocation(`20060102_150405`, simpleRegTimeInPkgName.FindString(pkg), time.Local)
|
||||
}
|
||||
|
||||
func (m *Manager) genStepPackage(base *vcs.Manifest, step Step) (newPkg string, newManifest *vcs.Manifest, err error) {
|
||||
defer func() {
|
||||
logrus.Infof("package dir %s for step %v generated", newPkg, step.IssueURLs)
|
||||
}()
|
||||
newManifest = clone.Clone(base).(*vcs.Manifest)
|
||||
for _, u := range step.StructureUpdates {
|
||||
if u.P2 != nil {
|
||||
newManifest.UpdateManifestProject(u.P2.Name, u.P2.Path, u.P2.Remote, u.P2.Revision, true)
|
||||
} else if u.P1 != nil {
|
||||
newManifest.RemoveManifestProject(u.P1.Name)
|
||||
}
|
||||
}
|
||||
for _, mr := range step.MRs {
|
||||
newManifest.UpdateManifestProject(mr.Repo, "", "", mr.SHA, false)
|
||||
}
|
||||
md5sum, err := newManifest.Standardize()
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(m.Workspace, md5sum), 0750); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(m.Workspace, md5sum, "__last_issue__"), []byte(fmt.Sprintf("%v", step.IssueURLs)), 0640); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
err = newManifest.WriteFile(filepath.Join(m.Workspace, md5sum, "manifest_tag.xml"))
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return md5sum, newManifest, nil
|
||||
}
|
||||
/*
|
||||
* Copyright (c) 2022 Huawei Device Co., Ltd.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package gitee_common
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"fotff/vcs"
|
||||
"fotff/vcs/gitee"
|
||||
"github.com/huandu/go-clone"
|
||||
"github.com/sirupsen/logrus"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type IssueInfo struct {
|
||||
visited bool
|
||||
RelatedIssues []string
|
||||
MRs []*gitee.Commit
|
||||
StructCTime string
|
||||
StructureUpdates []*vcs.ProjectUpdate
|
||||
}
|
||||
|
||||
type Step struct {
|
||||
IssueURLs []string
|
||||
MRs []*gitee.Commit
|
||||
StructCTime string
|
||||
StructureUpdates []*vcs.ProjectUpdate
|
||||
}
|
||||
|
||||
func (m *Manager) stepsFromGitee(from, to string) (pkgs []string, err error) {
|
||||
updates, err := m.getRepoUpdates(from, to)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
startTime, err := parseTime(from)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
endTime, err := parseTime(to)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logrus.Infof("find %d repo updates from %s to %s", len(updates), from, to)
|
||||
steps, err := getAllStepsFromGitee(startTime, endTime, m.Branch, updates)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logrus.Infof("find total %d steps from %s to %s", len(steps), from, to)
|
||||
baseManifest, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, from, "manifest_tag.xml"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, step := range steps {
|
||||
var newPkg string
|
||||
if newPkg, baseManifest, err = m.genStepPackage(baseManifest, step); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pkgs = append(pkgs, newPkg)
|
||||
}
|
||||
return pkgs, nil
|
||||
}
|
||||
|
||||
func (m *Manager) getRepoUpdates(from, to string) (updates []vcs.ProjectUpdate, err error) {
|
||||
m1, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, from, "manifest_tag.xml"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m2, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, to, "manifest_tag.xml"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return vcs.GetRepoUpdates(m1, m2)
|
||||
}
|
||||
|
||||
func getAllStepsFromGitee(startTime, endTime time.Time, branch string, updates []vcs.ProjectUpdate) (ret []Step, err error) {
|
||||
allMRs, err := getAllMRs(startTime, endTime, branch, updates)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
issueInfos, err := combineMRsToIssue(allMRs, branch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return combineIssuesToStep(issueInfos)
|
||||
}
|
||||
|
||||
func getAllMRs(startTime, endTime time.Time, branch string, updates []vcs.ProjectUpdate) (allMRs []*gitee.Commit, err error) {
|
||||
var once sync.Once
|
||||
for _, update := range updates {
|
||||
var prs []*gitee.Commit
|
||||
if update.P1.StructureDiff(update.P2) {
|
||||
once.Do(func() {
|
||||
prs, err = gitee.GetBetweenTimeMRs("openharmony", "manifest", branch, startTime, endTime)
|
||||
})
|
||||
if update.P1 != nil {
|
||||
var p1 []*gitee.Commit
|
||||
p1, err = gitee.GetBetweenTimeMRs("openharmony", update.P1.Name, branch, startTime, endTime)
|
||||
prs = append(prs, p1...)
|
||||
}
|
||||
if update.P2 != nil {
|
||||
var p2 []*gitee.Commit
|
||||
p2, err = gitee.GetBetweenTimeMRs("openharmony", update.P2.Name, branch, startTime, endTime)
|
||||
prs = append(prs, p2...)
|
||||
}
|
||||
} else {
|
||||
prs, err = gitee.GetBetweenMRs(gitee.CompareParam{
|
||||
Head: update.P2.Revision,
|
||||
Base: update.P1.Revision,
|
||||
Owner: "openharmony",
|
||||
Repo: update.P2.Name,
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allMRs = append(allMRs, prs...)
|
||||
}
|
||||
logrus.Infof("find total %d merge request commits of all repo updates", len(allMRs))
|
||||
return
|
||||
}
|
||||
|
||||
func combineMRsToIssue(allMRs []*gitee.Commit, branch string) (map[string]*IssueInfo, error) {
|
||||
ret := make(map[string]*IssueInfo)
|
||||
for _, mr := range allMRs {
|
||||
num, err := strconv.Atoi(strings.Trim(regexp.MustCompile(`!\d+ `).FindString(mr.Commit.Message), "! "))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse MR message for %s fail: %s", mr.URL, err)
|
||||
}
|
||||
issues, err := gitee.GetMRIssueURL(mr.Owner, mr.Repo, num)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(issues) == 0 {
|
||||
issues = []string{mr.URL}
|
||||
}
|
||||
var scs []*vcs.ProjectUpdate
|
||||
var scTime string
|
||||
if mr.Owner == "openharmony" && mr.Repo == "manifest" {
|
||||
if scTime, scs, err = parseStructureUpdates(mr, branch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for i, issue := range issues {
|
||||
if _, ok := ret[issue]; !ok {
|
||||
ret[issue] = &IssueInfo{
|
||||
MRs: []*gitee.Commit{mr},
|
||||
RelatedIssues: append(issues[:i], issues[i+1:]...),
|
||||
StructCTime: scTime,
|
||||
StructureUpdates: scs,
|
||||
}
|
||||
} else {
|
||||
ret[issue] = &IssueInfo{
|
||||
MRs: append(ret[issue].MRs, mr),
|
||||
RelatedIssues: append(ret[issue].RelatedIssues, append(issues[:i], issues[i+1:]...)...),
|
||||
StructCTime: scTime,
|
||||
StructureUpdates: append(ret[issue].StructureUpdates, scs...),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
logrus.Infof("find total %d issues of all repo updates", len(ret))
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func combineOtherRelatedIssue(parent, self *IssueInfo, all map[string]*IssueInfo) {
|
||||
if self.visited {
|
||||
return
|
||||
}
|
||||
self.visited = true
|
||||
for _, other := range self.RelatedIssues {
|
||||
if son, ok := all[other]; ok {
|
||||
combineOtherRelatedIssue(self, son, all)
|
||||
delete(all, other)
|
||||
}
|
||||
}
|
||||
parent.RelatedIssues = deDupIssues(append(parent.RelatedIssues, self.RelatedIssues...))
|
||||
parent.MRs = deDupMRs(append(parent.MRs, self.MRs...))
|
||||
parent.StructureUpdates = deDupProjectUpdates(append(parent.StructureUpdates, self.StructureUpdates...))
|
||||
if len(parent.StructCTime) != 0 && parent.StructCTime < self.StructCTime {
|
||||
parent.StructCTime = self.StructCTime
|
||||
}
|
||||
}
|
||||
|
||||
func deDupProjectUpdates(us []*vcs.ProjectUpdate) (retMRs []*vcs.ProjectUpdate) {
|
||||
dupIndexes := make([]bool, len(us))
|
||||
for i := range us {
|
||||
for j := i + 1; j < len(us); j++ {
|
||||
if us[j].P1 == us[i].P1 && us[j].P2 == us[i].P2 {
|
||||
dupIndexes[j] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, dup := range dupIndexes {
|
||||
if dup {
|
||||
continue
|
||||
}
|
||||
retMRs = append(retMRs, us[i])
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func deDupMRs(mrs []*gitee.Commit) (retMRs []*gitee.Commit) {
|
||||
tmp := make(map[string]*gitee.Commit)
|
||||
for _, m := range mrs {
|
||||
tmp[m.SHA] = m
|
||||
}
|
||||
for _, m := range tmp {
|
||||
retMRs = append(retMRs, m)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func deDupIssues(issues []string) (retIssues []string) {
|
||||
tmp := make(map[string]string)
|
||||
for _, i := range issues {
|
||||
tmp[i] = i
|
||||
}
|
||||
for _, i := range tmp {
|
||||
retIssues = append(retIssues, i)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// parseStructureUpdates get changed XMLs and parse it to recognize repo structure changes.
|
||||
// Since we do not care which revision a repo was, P1 is not welly handled, just assign it not nil for performance.
|
||||
func parseStructureUpdates(commit *gitee.Commit, branch string) (string, []*vcs.ProjectUpdate, error) {
|
||||
tmp := make(map[string]vcs.ProjectUpdate)
|
||||
if len(commit.Files) == 0 {
|
||||
// commit that queried from MR req does not contain file details, should fetch again
|
||||
var err error
|
||||
if commit, err = gitee.GetCommit(commit.Owner, commit.Repo, commit.SHA); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
for _, f := range commit.Files {
|
||||
if filepath.Ext(f.Filename) != ".xml" {
|
||||
continue
|
||||
}
|
||||
if err := parseFilePatch(f.Patch, tmp); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
var ret []*vcs.ProjectUpdate
|
||||
for _, pu := range tmp {
|
||||
projectUpdateCopy := pu
|
||||
ret = append(ret, &projectUpdateCopy)
|
||||
}
|
||||
for _, pu := range ret {
|
||||
if pu.P1 == nil && pu.P2 != nil {
|
||||
lastCommit, err := gitee.GetLatestMRBefore("openharmony", pu.P2.Name, branch, commit.Commit.Committer.Date)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
pu.P2.Revision = lastCommit.SHA
|
||||
}
|
||||
}
|
||||
return commit.Commit.Committer.Date, ret, nil
|
||||
}
|
||||
|
||||
func parseFilePatch(str string, m map[string]vcs.ProjectUpdate) error {
|
||||
sc := bufio.NewScanner(bytes.NewBuffer([]byte(str)))
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
var p vcs.Project
|
||||
if strings.HasPrefix(line, "-") {
|
||||
if err := xml.Unmarshal([]byte(line[1:]), &p); err == nil {
|
||||
m[p.Name] = vcs.ProjectUpdate{P1: &p, P2: m[p.Name].P2}
|
||||
}
|
||||
} else if strings.HasPrefix(line, "+") {
|
||||
if err := xml.Unmarshal([]byte(line[1:]), &p); err == nil {
|
||||
m[p.Name] = vcs.ProjectUpdate{P1: m[p.Name].P1, P2: &p}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func combineIssuesToStep(issueInfos map[string]*IssueInfo) (ret []Step, err error) {
|
||||
for _, info := range issueInfos {
|
||||
combineOtherRelatedIssue(info, info, issueInfos)
|
||||
}
|
||||
for issue, infos := range issueInfos {
|
||||
sort.Slice(infos.MRs, func(i, j int) bool {
|
||||
// move the latest MR to the first place, use its merged_time to represent the update time of the issue
|
||||
return infos.MRs[i].Commit.Committer.Date > infos.MRs[j].Commit.Committer.Date
|
||||
})
|
||||
ret = append(ret, Step{
|
||||
IssueURLs: append(infos.RelatedIssues, issue),
|
||||
MRs: infos.MRs,
|
||||
StructCTime: infos.StructCTime,
|
||||
StructureUpdates: infos.StructureUpdates})
|
||||
}
|
||||
sort.Slice(ret, func(i, j int) bool {
|
||||
ti, tj := ret[i].MRs[0].Commit.Committer.Date, ret[j].MRs[0].Commit.Committer.Date
|
||||
if len(ret[i].StructCTime) != 0 {
|
||||
ti = ret[i].StructCTime
|
||||
}
|
||||
if len(ret[j].StructCTime) != 0 {
|
||||
ti = ret[j].StructCTime
|
||||
}
|
||||
return ti < tj
|
||||
})
|
||||
logrus.Infof("find total %d steps of all issues", len(ret))
|
||||
return
|
||||
}
|
||||
|
||||
func parseTime(pkg string) (time.Time, error) {
|
||||
t, err := time.ParseInLocation(`20060102_150405`, regexp.MustCompile(`\d{8}_\d{6}`).FindString(pkg), time.Local)
|
||||
if err != nil {
|
||||
return time.ParseInLocation(`20060102150405`, regexp.MustCompile(`\d{14}`).FindString(pkg), time.Local)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (m *Manager) genStepPackage(base *vcs.Manifest, step Step) (newPkg string, newManifest *vcs.Manifest, err error) {
|
||||
defer func() {
|
||||
logrus.Infof("package dir %s for step %v generated", newPkg, step.IssueURLs)
|
||||
}()
|
||||
newManifest = clone.Clone(base).(*vcs.Manifest)
|
||||
for _, u := range step.StructureUpdates {
|
||||
if u.P2 != nil {
|
||||
newManifest.UpdateManifestProject(u.P2.Name, u.P2.Path, u.P2.Remote, u.P2.Revision, true)
|
||||
} else if u.P1 != nil {
|
||||
newManifest.RemoveManifestProject(u.P1.Name)
|
||||
}
|
||||
}
|
||||
for _, mr := range step.MRs {
|
||||
newManifest.UpdateManifestProject(mr.Repo, "", "", mr.SHA, false)
|
||||
}
|
||||
md5sum, err := newManifest.Standardize()
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(m.Workspace, md5sum), 0750); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(m.Workspace, md5sum, "__last_issue__"), []byte(fmt.Sprintf("%v", step.IssueURLs)), 0640); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
err = newManifest.WriteFile(filepath.Join(m.Workspace, md5sum, "manifest_tag.xml"))
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return md5sum, newManifest, nil
|
||||
}
|
||||
@ -13,7 +13,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package dayu200
|
||||
package gitee_common
|
||||
|
||||
import (
|
||||
"fotff/vcs"
|
||||
@ -17,10 +17,6 @@ package pkg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/sirupsen/logrus"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
type NewFunc func() Manager
|
||||
@ -37,25 +33,3 @@ type Manager interface {
|
||||
// PkgDir returns where pkg exists in the filesystem.
|
||||
PkgDir(pkg string) string
|
||||
}
|
||||
|
||||
func GetNewerFileFromDir(dir string, cur string, less func(files []os.DirEntry, i, j int) bool) string {
|
||||
for {
|
||||
files, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
logrus.Errorf("read dir %s err: %s", dir, err)
|
||||
time.Sleep(10 * time.Second)
|
||||
continue
|
||||
}
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return less(files, i, j)
|
||||
})
|
||||
if len(files) != 0 {
|
||||
f := files[len(files)-1]
|
||||
if f.Name() != cur {
|
||||
logrus.Infof("new package found, name: %s", f.Name())
|
||||
return f.Name()
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
@ -146,18 +146,7 @@ func flashAndTest(m pkg.Manager, t tester.Tester, pkg string, testcase string, c
|
||||
device := res.GetDevice()
|
||||
defer res.ReleaseDevice(device)
|
||||
if err := m.Flash(device, pkg, ctx); err != nil && !errors.Is(err, context.Canceled) {
|
||||
// Sometimes we need to find out the first compilation failure. Treat it as a normal test failure to re-use this framework.
|
||||
var cfg struct {
|
||||
AllowBuildError string `key:"allow_build_err"`
|
||||
}
|
||||
utils.ParseFromConfigFile("", &cfg)
|
||||
if cfg.AllowBuildError != "true" {
|
||||
return false, newFellows, err
|
||||
}
|
||||
logrus.Warnf("can not flash %s to %s, assume it as a failure: %v", pkg, device, err)
|
||||
for _, cases := range append(fellows, testcase) {
|
||||
results = append(results, tester.Result{TestCaseName: cases, Status: tester.ResultFail})
|
||||
}
|
||||
return false, newFellows, err
|
||||
} else {
|
||||
if err = t.Prepare(m.PkgDir(pkg), device, ctx); err != nil {
|
||||
return false, newFellows, err
|
||||
|
||||
81
tools/fotff/tester/pkg_available/pkg_available.go
Normal file
81
tools/fotff/tester/pkg_available/pkg_available.go
Normal file
@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright (c) 2023 Huawei Device Co., Ltd.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package pkg_available
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fotff/tester"
|
||||
"github.com/sirupsen/logrus"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Tester struct {
|
||||
device2PkgDir sync.Map
|
||||
}
|
||||
|
||||
func init() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func NewTester() tester.Tester {
|
||||
ret := &Tester{}
|
||||
return ret
|
||||
}
|
||||
|
||||
func (t *Tester) TaskName() string {
|
||||
return "pkg_available"
|
||||
}
|
||||
|
||||
func (t *Tester) Prepare(pkgDir string, device string, ctx context.Context) error {
|
||||
t.device2PkgDir.Store(device, pkgDir)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Tester) DoTestTask(deviceSN string, ctx context.Context) (ret []tester.Result, err error) {
|
||||
return t.DoTestCases(deviceSN, []string{"pkg_available"}, ctx)
|
||||
}
|
||||
|
||||
func (t *Tester) DoTestCase(deviceSN, testCase string, ctx context.Context) (ret tester.Result, err error) {
|
||||
pkgDir, _ := t.device2PkgDir.Load(deviceSN)
|
||||
es, err := os.ReadDir(pkgDir.(string))
|
||||
if err != nil {
|
||||
logrus.Errorf("can not read dir %s, testcase failed", pkgDir.(string))
|
||||
return tester.Result{TestCaseName: testCase, Status: tester.ResultFail}, nil
|
||||
}
|
||||
for _, e := range es {
|
||||
if strings.HasSuffix(e.Name(), ".img") {
|
||||
logrus.Infof("find image in dir %s, package is avaliable, testcase pass", pkgDir.(string))
|
||||
return tester.Result{TestCaseName: testCase, Status: tester.ResultPass}, nil
|
||||
}
|
||||
}
|
||||
logrus.Infof("no images in dir %s, package is not avaliable, testcase failed", pkgDir.(string))
|
||||
return tester.Result{TestCaseName: testCase, Status: tester.ResultFail}, nil
|
||||
}
|
||||
|
||||
func (t *Tester) DoTestCases(deviceSN string, testcases []string, ctx context.Context) (ret []tester.Result, err error) {
|
||||
for _, testcase := range testcases {
|
||||
r, err := t.DoTestCase(deviceSN, testcase, ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ret = append(ret, r)
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
@ -25,7 +25,11 @@ import (
|
||||
)
|
||||
|
||||
func DoSimpleHttpReqRaw(method string, url string, body []byte, header map[string]string) (response *http.Response, err error) {
|
||||
for i := 0; i < 3; i++ {
|
||||
maxRetry := len(proxyList)
|
||||
if maxRetry < 3 {
|
||||
maxRetry = 3
|
||||
}
|
||||
for i := 0; i < maxRetry; i++ {
|
||||
if response, err = doSimpleHttpReqImpl(method, url, body, header); err == nil {
|
||||
return
|
||||
}
|
||||
@ -36,7 +40,11 @@ func DoSimpleHttpReqRaw(method string, url string, body []byte, header map[strin
|
||||
|
||||
func DoSimpleHttpReq(method string, url string, body []byte, header map[string]string) (ret []byte, err error) {
|
||||
var resp *http.Response
|
||||
for i := 0; i < 3; i++ {
|
||||
maxRetry := len(proxyList)
|
||||
if maxRetry < 3 {
|
||||
maxRetry = 3
|
||||
}
|
||||
for i := 0; i < maxRetry; i++ {
|
||||
if resp, err = doSimpleHttpReqImpl(method, url, body, header); err == nil {
|
||||
ret, err = io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
|
||||
@ -19,6 +19,7 @@ import (
|
||||
"github.com/Unknwon/goconfig"
|
||||
"github.com/sirupsen/logrus"
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ParseFromConfigFile parse ini file and set values by the tag of fields.
|
||||
@ -46,6 +47,22 @@ func ParseFromConfigFile(section string, p any) {
|
||||
v = rt.Elem().Field(i).Tag.Get("default")
|
||||
}
|
||||
rv.Elem().Field(i).SetString(v)
|
||||
case reflect.Slice:
|
||||
if rt.Elem().Field(i).Type.Elem().Kind() != reflect.String {
|
||||
break
|
||||
}
|
||||
key := rt.Elem().Field(i).Tag.Get("key")
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
var v string
|
||||
if conf != nil {
|
||||
v, err = conf.GetValue(section, key)
|
||||
}
|
||||
if conf == nil || err != nil {
|
||||
v = rt.Elem().Field(i).Tag.Get("default")
|
||||
}
|
||||
rv.Elem().Field(i).Set(reflect.ValueOf(strings.Split(v, ",")))
|
||||
case reflect.Struct:
|
||||
ParseFromConfigFile(section, rv.Elem().Field(i).Addr().Interface())
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user