Code Monkey home page Code Monkey logo

Comments (22)

rmb938 avatar rmb938 commented on August 24, 2024 1

Ok will do

from go-diskfs.

rmb938 avatar rmb938 commented on August 24, 2024 1

Here you go, same commands ran in the same order except with mkfs.vfat -F 32 -s 8 /dev/sda1

manual.img.gz

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

I was able to recreate it. Slightly changed code to make it easier to create each one without changing code.

package main

import (
	"fmt"
	"log"
	"os"
	"path"
	"strconv"

	diskfs "github.com/diskfs/go-diskfs"
	"github.com/diskfs/go-diskfs/disk"
	"github.com/diskfs/go-diskfs/filesystem"
	"github.com/diskfs/go-diskfs/partition/mbr"
)

func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func CreateFSAndDir(diskImg string, partSize int) int64 {
	if diskImg == "" {
		log.Fatal("must have a valid path for diskImg")
	}
	mydisk, err := diskfs.Open(diskImg)
	if err != nil {
		var diskSize int64
		diskSize = 5 * 1024 * 1024 * 1024 // 5 GB
		mydisk, err = diskfs.Create(diskImg, diskSize, diskfs.Raw)
		check(err)
	}

	cloudInitSize := partSize * 1024 * 1024 * 1024 // 1 GB
	cloudInitSectors := uint32(cloudInitSize / 512)
	// we want to create it at the end of the disk
	// so find the disk sector count and minus the cloudinit sectors
	cloudInitStart := uint32(int(mydisk.Size)/512) - cloudInitSectors

	table := &mbr.Table{
		LogicalSectorSize:  512,
		PhysicalSectorSize: 512,
		Partitions: []*mbr.Partition{
			{
				Bootable: false,
				Type:     mbr.Linux,
				Start:    cloudInitStart,
				Size:     cloudInitSectors,
			},
		},
	}

	log.Print("Writing partition table to disk")
	err = mydisk.Partition(table)
	check(err)

	fspec := disk.FilesystemSpec{Partition: 1, FSType: filesystem.TypeFat32, VolumeLabel: "config-2"}
	fs, err := mydisk.CreateFilesystem(fspec)
	check(err)

	cloudInitPrefix := path.Join("/", "openstack", "latest")
	// place down cloud-init info
	log.Print("Creating cloud init directory structure")
	err = fs.Mkdir(cloudInitPrefix)
	if err != nil {
		log.Fatalf("Error creating cloud init directory structure: %v", err)
	}
	return int64(cloudInitStart)*512
}

func main() {
	if len(os.Args) < 3 {
		fmt.Printf("Usage: %s <outfile> <partition size in GB>\n", os.Args[0])
		fmt.Printf("Example: %s ./foo.img 2\n", os.Args[0])
		os.Exit(1)
	}
	i, err := strconv.Atoi(os.Args[2])
	if err != nil {
		fmt.Printf("second argument *must* be an integer, you gave %s: %v", os.Args[1], err)
		os.Exit(1)
	}

	start := CreateFSAndDir(os.Args[1], i)
	fmt.Printf("fat32 partition starts at byte %d\n", start)
}

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

Here is another variant that actually reads the filesystem:

package main

import (
	"fmt"
	"log"
	"os"
	"path"
	"strconv"

	diskfs "github.com/diskfs/go-diskfs"
	"github.com/diskfs/go-diskfs/disk"
	"github.com/diskfs/go-diskfs/filesystem"
	"github.com/diskfs/go-diskfs/partition/mbr"
)

func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func CreateFSAndDir(diskImg string, partSize int) int64 {
	if diskImg == "" {
		log.Fatal("must have a valid path for diskImg")
	}
	mydisk, err := diskfs.Open(diskImg)
	if err != nil {
		var diskSize int64
		diskSize = 5 * 1024 * 1024 * 1024 // 5 GB
		mydisk, err = diskfs.Create(diskImg, diskSize, diskfs.Raw)
		check(err)
	}

	cloudInitSize := partSize * 1024 * 1024 * 1024 // 1 GB
	cloudInitSectors := uint32(cloudInitSize / 512)
	// we want to create it at the end of the disk
	// so find the disk sector count and minus the cloudinit sectors
	cloudInitStart := uint32(int(mydisk.Size)/512) - cloudInitSectors

	table := &mbr.Table{
		LogicalSectorSize:  512,
		PhysicalSectorSize: 512,
		Partitions: []*mbr.Partition{
			{
				Bootable: false,
				Type:     mbr.Linux,
				Start:    cloudInitStart,
				Size:     cloudInitSectors,
			},
		},
	}

	log.Print("Writing partition table to disk")
	err = mydisk.Partition(table)
	check(err)

	fspec := disk.FilesystemSpec{Partition: 1, FSType: filesystem.TypeFat32, VolumeLabel: "config-2"}
	fs, err := mydisk.CreateFilesystem(fspec)
	check(err)

	cloudInitPrefix := path.Join("/", "openstack", "latest")
	// place down cloud-init info
	log.Print("Creating cloud init directory structure")
	err = fs.Mkdir(cloudInitPrefix)
	if err != nil {
		log.Fatalf("Error creating cloud init directory structure: %v", err)
	}
	return int64(cloudInitStart)*512
}

func ReadRootDir(diskImg string) []os.FileInfo {
	disk, err := diskfs.Open(diskImg)
	if err != nil {
		log.Panic(err)
	}
	table, err := disk.GetPartitionTable()
	if err != nil {
		log.Panic(err)
	}
	disk.Table = table
	fs, err := disk.GetFilesystem(1)
	if err != nil {
		log.Panic(err)
	}
	files, err := fs.ReadDir("/")
	if err != nil {
		log.Panic(err)
	}
	return files

}

func main() {
	if len(os.Args) < 3 {
		fmt.Printf("Usage: %s <outfile> <partition size in GB>\n", os.Args[0])
		fmt.Printf("Example: %s ./foo.img 2\n", os.Args[0])
		os.Exit(1)
	}
	i, err := strconv.Atoi(os.Args[2])
	if err != nil {
		fmt.Printf("second argument *must* be an integer, you gave %s: %v", os.Args[1], err)
		os.Exit(1)
	}

	start := CreateFSAndDir(os.Args[1], i)
	fmt.Printf("fat32 partition starts at byte %d\n", start)
	root := ReadRootDir(os.Args[1])
	fmt.Printf("%#v\n", root)
}

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

The funny part is the above succeeds in reading it. This makes it much more difficult, as it is not clear where the failure is.

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

Here’s an idea. Can you make a 5/1 image (the kind that fails) via the library, another via manual tools (and verify that works). I then can bite compare them.

from go-diskfs.

rmb938 avatar rmb938 commented on August 24, 2024

Will do

from go-diskfs.

rmb938 avatar rmb938 commented on August 24, 2024

Here are the images, same naming as before

images.tar.gz

go-diskfs

[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# go run main.go /dev/sda 
2020/01/29 22:27:49 Writing partition table to disk
2020/01/29 22:27:49 Creating cloud init directory structure

[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# fdisk -lu /dev/sda
Disk /dev/sda: 5 GiB, 5368709120 bytes, 10485760 sectors
Disk model: Volume          
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: dos
Disk identifier: 0x00000000

Device     Boot   Start      End Sectors Size Id Type
/dev/sda1       8388608 10485759 2097152   1G 83 Linux

[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# mount /dev/sda1 /mnt

[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# lsblk
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda      8:0    0    5G  0 disk 
└─sda1   8:1    0    1G  0 part /mnt
vda    252:0    0  160G  0 disk 
└─vda1 252:1    0  160G  0 part /
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# blkid
/dev/vda1: UUID="b1438b9b-2cab-4065-a99a-08a96687f73c" TYPE="ext4" PARTUUID="7a0392d3-01"
/dev/sda1: LABEL_FATBOOT="config-2" UUID="4EF0-7AFF" TYPE="vfat"
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# umount /mnt

[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# dd if=/dev/sda of=go-diskfs.img
10485760+0 records in
10485760+0 records out
5368709120 bytes (5.4 GB, 5.0 GiB) copied, 51.4466 s, 104 MB/s

manual

[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# fdisk /dev/sda 

Welcome to fdisk (util-linux 2.34).
Changes will remain in memory only, until you decide to write them.
Be careful before using the write command.

Device does not contain a recognized partition table.
Created a new DOS disklabel with disk identifier 0x6b97644a.

Command (m for help): n
Partition type
   p   primary (0 primary, 0 extended, 4 free)
   e   extended (container for logical partitions)
Select (default p): 

Using default response p.
Partition number (1-4, default 1): 
First sector (2048-10485759, default 2048): 8388608
Last sector, +/-sectors or +/-size{K,M,G,T,P} (8388608-10485759, default 10485759): 10485759

Created a new partition 1 of type 'Linux' and of size 1 GiB.

Command (m for help): p
Disk /dev/sda: 5 GiB, 5368709120 bytes, 10485760 sectors
Disk model: Volume          
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: dos
Disk identifier: 0x6b97644a

Device     Boot   Start      End Sectors Size Id Type
/dev/sda1       8388608 10485759 2097152   1G 83 Linux

Command (m for help): w
The partition table has been altered.
Calling ioctl() to re-read partition table.
Syncing disks.

[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# mkfs.vfat -F 32 -s 2 /dev/sda1
mkfs.fat 4.1 (2017-01-24)
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# fatlabel /dev/sda1 config-2
fatlabel: warning - lowercase labels might not work properly with DOS or Windows
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# mount /dev/sda1 /mnt
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# mkdir -p /mnt/openstack/latest
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# ls -la /mnt/
total 6
drwxr-xr-x.  3 root root 1024 Jan  1  1970 .
dr-xr-xr-x. 18 root root 4096 Jan 29 14:43 ..
drwxr-xr-x.  3 root root 1024 Jan 29 22:33 openstack
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# lsblk
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda      8:0    0    5G  0 disk 
└─sda1   8:1    0    1G  0 part /mnt
vda    252:0    0  160G  0 disk 
└─vda1 252:1    0  160G  0 part /
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# blkid
/dev/vda1: UUID="b1438b9b-2cab-4065-a99a-08a96687f73c" TYPE="ext4" PARTUUID="7a0392d3-01"
/dev/sda1: LABEL_FATBOOT="config-2" LABEL="config-2" UUID="804C-E019" TYPE="vfat" PARTUUID="6b97644a-01"
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# umount /mnt
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# dd if=/dev/sda of=manual.img
10485760+0 records in
10485760+0 records out
5368709120 bytes (5.4 GB, 5.0 GiB) copied, 46.7378 s, 115 MB/s

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

That definitely helped. I removed the partition from the image using dd, and then tried to mount just the partition file. I got the same response, both on Linux and on Mac. This indicates that it isn't some odd partitioning thing, but something to do with the calculation of sizes for the filesystem in the partition itself, and makes it easier to work with.

FWIW:

$ dd skip=8388608 bs=512 if=./go-diskfs.img  of=./go-diskfs-partition.img
$ dd skip=8388608 bs=512 if=./manual.img of=./manual-partition.img

And then just mount go-diskfs-partition.img and manual-partition.img.

from go-diskfs.

rmb938 avatar rmb938 commented on August 24, 2024

Awesome, glad you can see a difference.

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

This is not going to be a fun one to debug, but I am cautiously optimistic.

from go-diskfs.

rmb938 avatar rmb938 commented on August 24, 2024

Let me know if you need me to generate any more images or try different things. I'm open to help in any way I can.

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

Actually, I am curious why, when you ran it manually, you overrode to set the sectors per cluster to 2 with -s 2?

from go-diskfs.

rmb938 avatar rmb938 commented on August 24, 2024

Umm I actually can't remember. I thought I saw somewhere in the go-diskfs code it setting it to 2 but I can't find that anymore... so maybe I just made it up lol

I can regenerate the manual image without setting -s 2

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

Or set it to 8 maybe? There is no official spec for it, but go-diskfs defaults to using what most of the Linux utilities nominally use. A lot of this is unofficial "dirty art/magic" stuff.

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

Thanks, that helped. I am seeing slight unexplained differences in the FAT tables.

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

@rmb938 can you validate #50? My tests show it works.

from go-diskfs.

rmb938 avatar rmb938 commented on August 24, 2024

So the fix works but now the drive label is showing up as a file

[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# go run main.go /dev/sda 
2020/01/31 14:31:54 Writing partition table to disk
2020/01/31 14:31:54 Creating cloud init directory structure
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# mount /dev/sda1 /mnt
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# ls -la /mnt
total 12
drwxr-xr-x.  3 root root 4096 Jan  1  1970 .
dr-xr-xr-x. 18 root root 4096 Jan 29 14:43 ..
-rwxr-xr-x.  1 root root    0 Jan 31 14:31 config-2
drwxr-xr-x.  3 root root 4096 Jan 31 14:31 openstack

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

good to know. We will get that.

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

That is an issue in the volume label creation, not in the size issue. I am going to merge that one in, then we can fix the other separately.

from go-diskfs.

deitch avatar deitch commented on August 24, 2024

@rmb938 check out #51 please.

from go-diskfs.

rmb938 avatar rmb938 commented on August 24, 2024

@deitch 👍 that fixed it. Thanks so much for all the help.

from go-diskfs.

Related Issues (20)

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. 📊📈🎉

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google ❤️ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.