Skip to content

Commit b1a2ec4

Browse files
pibigstarpibigstar
committed
fix the cpuset and add pivot root
1 parent 05f1ed9 commit b1a2ec4

File tree

9 files changed

+139
-42
lines changed

9 files changed

+139
-42
lines changed

‎README.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,4 +57,9 @@ mount -t cgroup -o none,name=cgroup-test cgroup-test ./cgroup-test
5757
> 只要将该进程的ID放到其`cgroup`的`tasks`里面即可
5858
```bash
5959
echo "进程ID" >> cgroup/tasks
60+
```
61+
62+
- 导出容器
63+
```bash
64+
docker export -o busybox.tar 45c98e055883(容器ID)
6065
```

‎cgroups/subsystem/cpu.go

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
)
1010

1111
type CpuSubSystem struct {
12+
apply bool
1213
}
1314

1415
func (*CpuSubSystem) Name() string {
@@ -22,6 +23,7 @@ func (c *CpuSubSystem) Set(cgroupPath string, res *ResourceConfig) error {
2223
return err
2324
}
2425
if res.CpuShare != "" {
26+
c.apply = true
2527
err = ioutil.WriteFile(path.Join(subsystemCgroupPath, "cpu.shares"), []byte(res.CpuShare), 0644)
2628
if err != nil {
2729
logrus.Errorf("failed to write file cpu.shares, err: %+v", err)
@@ -32,24 +34,26 @@ func (c *CpuSubSystem) Set(cgroupPath string, res *ResourceConfig) error {
3234
}
3335

3436
func (c *CpuSubSystem) Remove(cgroupPath string) error {
35-
subsystemCgroupPath, err := GetCgroupPath(c.Name(), cgroupPath, true)
37+
subsystemCgroupPath, err := GetCgroupPath(c.Name(), cgroupPath, false)
3638
if err != nil {
3739
return err
3840
}
3941
return os.RemoveAll(subsystemCgroupPath)
4042
}
4143

4244
func (c *CpuSubSystem) Apply(cgroupPath string, pid int) error {
43-
subsystemCgroupPath, err := GetCgroupPath(c.Name(), cgroupPath, true)
44-
if err != nil {
45-
return err
46-
}
45+
if c.apply {
46+
subsystemCgroupPath, err := GetCgroupPath(c.Name(), cgroupPath, false)
47+
if err != nil {
48+
return err
49+
}
4750

48-
tasksPath := path.Join(subsystemCgroupPath, "tasks")
49-
err = ioutil.WriteFile(tasksPath, []byte(strconv.Itoa(pid)), 0644)
50-
if err != nil {
51-
logrus.Errorf("write pid to tasks, path: %s, pid: %d, err: %v", tasksPath, pid, err)
52-
return err
51+
tasksPath := path.Join(subsystemCgroupPath, "tasks")
52+
err = ioutil.WriteFile(tasksPath, []byte(strconv.Itoa(pid)), os.ModePerm)
53+
if err != nil {
54+
logrus.Errorf("write pid to tasks, path: %s, pid: %d, err: %v", tasksPath, pid, err)
55+
return err
56+
}
5357
}
5458
return nil
5559
}

‎cgroups/subsystem/cpuset.go

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
package subsystem
22

33
import (
4+
"github.com/sirupsen/logrus"
45
"io/ioutil"
56
"os"
67
"path"
78
"strconv"
8-
9-
"github.com/sirupsen/logrus"
109
)
1110

1211
type CpuSetSubSystem struct {
12+
apply bool
1313
}
1414

1515
func (*CpuSetSubSystem) Name() string {
@@ -23,6 +23,7 @@ func (c *CpuSetSubSystem) Set(cgroupPath string, res *ResourceConfig) error {
2323
return err
2424
}
2525
if res.CpuSet != "" {
26+
c.apply = true
2627
err := ioutil.WriteFile(path.Join(subsystemCgroupPath, "cpuset.cpus"), []byte(res.CpuSet), 0644)
2728
if err != nil {
2829
logrus.Errorf("failed to write file cpuset.cpus, err: %+v", err)
@@ -33,23 +34,25 @@ func (c *CpuSetSubSystem) Set(cgroupPath string, res *ResourceConfig) error {
3334
}
3435

3536
func (c *CpuSetSubSystem) Remove(cgroupPath string) error {
36-
subsystemCgroupPath, err := GetCgroupPath(c.Name(), cgroupPath, true)
37+
subsystemCgroupPath, err := GetCgroupPath(c.Name(), cgroupPath, false)
3738
if err != nil {
3839
return err
3940
}
4041
return os.RemoveAll(subsystemCgroupPath)
4142
}
4243

4344
func (c *CpuSetSubSystem) Apply(cgroupPath string, pid int) error {
44-
subsystemCgroupPath, err := GetCgroupPath(c.Name(), cgroupPath, true)
45-
if err != nil {
46-
return err
47-
}
48-
tasksPath := path.Join(subsystemCgroupPath, "tasks")
49-
err = ioutil.WriteFile(tasksPath, []byte(strconv.Itoa(pid)), 0644)
50-
if err != nil {
51-
logrus.Errorf("write pid to tasks, path: %s, pid: %d, err: %v", tasksPath, pid, err)
52-
return err
45+
if c.apply {
46+
subsystemCgroupPath, err := GetCgroupPath(c.Name(), cgroupPath, false)
47+
if err != nil {
48+
return err
49+
}
50+
tasksPath := path.Join(subsystemCgroupPath, "tasks")
51+
err = ioutil.WriteFile(tasksPath, []byte(strconv.Itoa(pid)), os.ModePerm)
52+
if err != nil {
53+
logrus.Errorf("write pid to tasks, path: %s, pid: %d, err: %v", tasksPath, pid, err)
54+
return err
55+
}
5356
}
5457
return nil
5558
}

‎cgroups/subsystem/memory.go

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
)
1111

1212
type MemorySubSystem struct {
13+
apply bool
1314
}
1415

1516
func (*MemorySubSystem) Name() string {
@@ -23,6 +24,7 @@ func (m *MemorySubSystem) Set(cgroupPath string, res *ResourceConfig) error {
2324
return err
2425
}
2526
if res.MemoryLimit != "" {
27+
m.apply = true
2628
// 设置cgroup内存限制，
2729
// 将这个限制写入到cgroup对应目录的 memory.limit_in_bytes文件中即可
2830
err := ioutil.WriteFile(path.Join(subsystemCgroupPath, "memory.limit_in_bytes"), []byte(res.MemoryLimit), 0644)
@@ -34,23 +36,25 @@ func (m *MemorySubSystem) Set(cgroupPath string, res *ResourceConfig) error {
3436
}
3537

3638
func (m *MemorySubSystem) Remove(cgroupPath string) error {
37-
subsystemCgroupPath, err := GetCgroupPath(m.Name(), cgroupPath, true)
39+
subsystemCgroupPath, err := GetCgroupPath(m.Name(), cgroupPath, false)
3840
if err != nil {
3941
return err
4042
}
4143
return os.RemoveAll(subsystemCgroupPath)
4244
}
4345

4446
func (m *MemorySubSystem) Apply(cgroupPath string, pid int) error {
45-
subsystemCgroupPath, err := GetCgroupPath(m.Name(), cgroupPath, true)
46-
if err != nil {
47-
return err
48-
}
49-
tasksPath := path.Join(subsystemCgroupPath, "tasks")
50-
err = ioutil.WriteFile(tasksPath, []byte(strconv.Itoa(pid)), 0644)
51-
if err != nil {
52-
logrus.Errorf("write pid to tasks, path: %s, pid: %d, err: %v", tasksPath, pid, err)
53-
return err
47+
if m.apply {
48+
subsystemCgroupPath, err := GetCgroupPath(m.Name(), cgroupPath, false)
49+
if err != nil {
50+
return err
51+
}
52+
tasksPath := path.Join(subsystemCgroupPath, "tasks")
53+
err = ioutil.WriteFile(tasksPath, []byte(strconv.Itoa(pid)), os.ModePerm)
54+
if err != nil {
55+
logrus.Errorf("write pid to tasks, path: %s, pid: %d, err: %v", tasksPath, pid, err)
56+
return err
57+
}
5458
}
5559
return nil
5660
}

‎cgroups/subsystem/subsystem.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ type Subystem interface {
2727
var (
2828
Subsystems = []Subystem{
2929
&MemorySubSystem{},
30+
// 设置tasks时，这两个必须同时设置
3031
&CpuSubSystem{},
3132
&CpuSetSubSystem{},
3233
}

‎cgroups/subsystem/util.go

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package subsystem
22

33
import (
44
"bufio"
5-
"fmt"
65
"os"
76
"path"
87
"strings"
@@ -19,15 +18,13 @@ func GetCgroupPath(subsystem string, cgroupPath string, autoCreate bool) (string
1918
}
2019
cgroupTotalPath := path.Join(cgroupRootPath, cgroupPath)
2120
_, err = os.Stat(cgroupTotalPath)
22-
if err == nil || (autoCreate && os.IsNotExist(err)) {
23-
if os.IsNotExist(err) {
24-
if err := os.MkdirAll(cgroupTotalPath, 0755); err != nil {
25-
return "", err
26-
}
21+
if err != nil && os.IsNotExist(err) {
22+
if err := os.MkdirAll(cgroupTotalPath, 0755); err != nil {
23+
return "", err
2724
}
28-
return cgroupTotalPath, nil
2925
}
30-
return "", fmt.Errorf("cgroup path error")
26+
27+
return cgroupTotalPath, nil
3128
}
3229

3330
// 找到挂载了 subsystem 的hierarchy cgroup根节点所在的目录

‎container/init.go

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"io/ioutil"
66
"os"
77
"os/exec"
8+
"path/filepath"
89
"strings"
910
"syscall"
1011

@@ -29,8 +30,7 @@ func RunContainerInitProcess() error {
2930
// 在系统环境 PATH中寻找命令的绝对路径
3031
path, err := exec.LookPath(cmdArray[0])
3132
if err != nil {
32-
logrus.Errorf("look %s path, err: %v", cmdArray[0], err)
33-
return err
33+
path = cmdArray[0]
3434
}
3535

3636
err = syscall.Exec(path, cmdArray[0:], os.Environ())
@@ -60,13 +60,67 @@ func setUpMount() error {
6060
if err != nil {
6161
return err
6262
}
63+
64+
err = pivotRoot()
65+
if err != nil {
66+
logrus.Errorf("pivot root, err: %v", err)
67+
return err
68+
}
69+
6370
//mount proc
6471
defaultMountFlags := syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV
6572
err = syscall.Mount("proc", "/proc", "proc", uintptr(defaultMountFlags), "")
6673
if err != nil {
6774
logrus.Errorf("mount proc, err: %v", err)
6875
return err
6976
}
77+
// mount temfs, temfs是一种基于内存的文件系统
78+
err = syscall.Mount("tmpfs", "/dev", "tmpfs", syscall.MS_NOSUID|syscall.MS_STRICTATIME, "mode=755")
79+
if err != nil {
80+
logrus.Errorf("mount tempfs, err: %v", err)
81+
return err
82+
}
7083

7184
return nil
7285
}
86+
87+
// 改变当前root的文件系统
88+
func pivotRoot() error {
89+
root, err := os.Getwd()
90+
if err != nil {
91+
return err
92+
}
93+
logrus.Infof("current location is %s", root)
94+
/**
95+
为了使当前root的老 root 和新 root 不在同一个文件系统下，我们把root重新mount了一次
96+
bind mount是把相同的内容换了一个挂载点的挂载方法
97+
*/
98+
if err := syscall.Mount(root, root, "bind", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
99+
return fmt.Errorf("mount rootfs to itself error: %v", err)
100+
}
101+
// 创建 rootfs/.pivot_root 存储 old_root
102+
pivotDir := filepath.Join(root, ".pivot_root")
103+
_, err = os.Stat(pivotDir)
104+
if err != nil && os.IsNotExist(err) {
105+
if err := os.Mkdir(pivotDir, 0777); err != nil {
106+
return err
107+
}
108+
}
109+
// pivot_root 到新的rootfs, 现在老的 old_root 是挂载在rootfs/.pivot_root
110+
// 挂载点现在依然可以在mount命令中看到
111+
if err := syscall.PivotRoot(root, pivotDir); err != nil {
112+
return fmt.Errorf("pivot_root %v", err)
113+
}
114+
// 修改当前的工作目录到根目录
115+
if err := syscall.Chdir("/"); err != nil {
116+
return fmt.Errorf("chdir / %v", err)
117+
}
118+
119+
pivotDir = filepath.Join("/", ".pivot_root")
120+
// unmount rootfs/.pivot_root
121+
if err := syscall.Unmount(pivotDir, syscall.MNT_DETACH); err != nil {
122+
return fmt.Errorf("unmount pivot_root dir %v", err)
123+
}
124+
// 删除临时文件夹
125+
return os.Remove(pivotDir)
126+
}

‎container/process.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,7 @@ func NewParentProcess(tty bool) (*exec.Cmd, *os.File) {
2323
cmd.ExtraFiles = []*os.File{
2424
readPipe,
2525
}
26+
// 指定容器初始化后的工作目录
27+
cmd.Dir = "/root/busybox"
2628
return cmd, writePipe
2729
}

‎test/util/util_test.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package util
2+
3+
import (
4+
"os/exec"
5+
"syscall"
6+
"testing"
7+
)
8+
9+
func TestLookPath(t *testing.T) {
10+
// 寻找 ls 命令的绝对路径
11+
path, err := exec.LookPath("ls")
12+
if err != nil {
13+
t.Error(err)
14+
}
15+
t.Logf("ls path: %s \n", path)
16+
}
17+
18+
// 切换运行时目录
19+
func TestChangeRunDir(t *testing.T) {
20+
err := syscall.Chdir("/root")
21+
if err != nil {
22+
t.Error(err)
23+
}
24+
cmd := exec.Command("pwd")
25+
bs, _ := cmd.CombinedOutput()
26+
t.Log(string(bs))
27+
}

0 commit comments

Comments
 (0)