Comments (22)
Ok will do
from go-diskfs.
Here you go, same commands ran in the same order except with mkfs.vfat -F 32 -s 8 /dev/sda1
from go-diskfs.
I was able to recreate it. Slightly changed code to make it easier to create each one without changing code.
package main
import (
"fmt"
"log"
"os"
"path"
"strconv"
diskfs "github.com/diskfs/go-diskfs"
"github.com/diskfs/go-diskfs/disk"
"github.com/diskfs/go-diskfs/filesystem"
"github.com/diskfs/go-diskfs/partition/mbr"
)
func check(err error) {
if err != nil {
log.Fatal(err)
}
}
func CreateFSAndDir(diskImg string, partSize int) int64 {
if diskImg == "" {
log.Fatal("must have a valid path for diskImg")
}
mydisk, err := diskfs.Open(diskImg)
if err != nil {
var diskSize int64
diskSize = 5 * 1024 * 1024 * 1024 // 5 GB
mydisk, err = diskfs.Create(diskImg, diskSize, diskfs.Raw)
check(err)
}
cloudInitSize := partSize * 1024 * 1024 * 1024 // 1 GB
cloudInitSectors := uint32(cloudInitSize / 512)
// we want to create it at the end of the disk
// so find the disk sector count and minus the cloudinit sectors
cloudInitStart := uint32(int(mydisk.Size)/512) - cloudInitSectors
table := &mbr.Table{
LogicalSectorSize: 512,
PhysicalSectorSize: 512,
Partitions: []*mbr.Partition{
{
Bootable: false,
Type: mbr.Linux,
Start: cloudInitStart,
Size: cloudInitSectors,
},
},
}
log.Print("Writing partition table to disk")
err = mydisk.Partition(table)
check(err)
fspec := disk.FilesystemSpec{Partition: 1, FSType: filesystem.TypeFat32, VolumeLabel: "config-2"}
fs, err := mydisk.CreateFilesystem(fspec)
check(err)
cloudInitPrefix := path.Join("/", "openstack", "latest")
// place down cloud-init info
log.Print("Creating cloud init directory structure")
err = fs.Mkdir(cloudInitPrefix)
if err != nil {
log.Fatalf("Error creating cloud init directory structure: %v", err)
}
return int64(cloudInitStart)*512
}
func main() {
if len(os.Args) < 3 {
fmt.Printf("Usage: %s <outfile> <partition size in GB>\n", os.Args[0])
fmt.Printf("Example: %s ./foo.img 2\n", os.Args[0])
os.Exit(1)
}
i, err := strconv.Atoi(os.Args[2])
if err != nil {
fmt.Printf("second argument *must* be an integer, you gave %s: %v", os.Args[1], err)
os.Exit(1)
}
start := CreateFSAndDir(os.Args[1], i)
fmt.Printf("fat32 partition starts at byte %d\n", start)
}
from go-diskfs.
Here is another variant that actually reads the filesystem:
package main
import (
"fmt"
"log"
"os"
"path"
"strconv"
diskfs "github.com/diskfs/go-diskfs"
"github.com/diskfs/go-diskfs/disk"
"github.com/diskfs/go-diskfs/filesystem"
"github.com/diskfs/go-diskfs/partition/mbr"
)
func check(err error) {
if err != nil {
log.Fatal(err)
}
}
func CreateFSAndDir(diskImg string, partSize int) int64 {
if diskImg == "" {
log.Fatal("must have a valid path for diskImg")
}
mydisk, err := diskfs.Open(diskImg)
if err != nil {
var diskSize int64
diskSize = 5 * 1024 * 1024 * 1024 // 5 GB
mydisk, err = diskfs.Create(diskImg, diskSize, diskfs.Raw)
check(err)
}
cloudInitSize := partSize * 1024 * 1024 * 1024 // 1 GB
cloudInitSectors := uint32(cloudInitSize / 512)
// we want to create it at the end of the disk
// so find the disk sector count and minus the cloudinit sectors
cloudInitStart := uint32(int(mydisk.Size)/512) - cloudInitSectors
table := &mbr.Table{
LogicalSectorSize: 512,
PhysicalSectorSize: 512,
Partitions: []*mbr.Partition{
{
Bootable: false,
Type: mbr.Linux,
Start: cloudInitStart,
Size: cloudInitSectors,
},
},
}
log.Print("Writing partition table to disk")
err = mydisk.Partition(table)
check(err)
fspec := disk.FilesystemSpec{Partition: 1, FSType: filesystem.TypeFat32, VolumeLabel: "config-2"}
fs, err := mydisk.CreateFilesystem(fspec)
check(err)
cloudInitPrefix := path.Join("/", "openstack", "latest")
// place down cloud-init info
log.Print("Creating cloud init directory structure")
err = fs.Mkdir(cloudInitPrefix)
if err != nil {
log.Fatalf("Error creating cloud init directory structure: %v", err)
}
return int64(cloudInitStart)*512
}
func ReadRootDir(diskImg string) []os.FileInfo {
disk, err := diskfs.Open(diskImg)
if err != nil {
log.Panic(err)
}
table, err := disk.GetPartitionTable()
if err != nil {
log.Panic(err)
}
disk.Table = table
fs, err := disk.GetFilesystem(1)
if err != nil {
log.Panic(err)
}
files, err := fs.ReadDir("/")
if err != nil {
log.Panic(err)
}
return files
}
func main() {
if len(os.Args) < 3 {
fmt.Printf("Usage: %s <outfile> <partition size in GB>\n", os.Args[0])
fmt.Printf("Example: %s ./foo.img 2\n", os.Args[0])
os.Exit(1)
}
i, err := strconv.Atoi(os.Args[2])
if err != nil {
fmt.Printf("second argument *must* be an integer, you gave %s: %v", os.Args[1], err)
os.Exit(1)
}
start := CreateFSAndDir(os.Args[1], i)
fmt.Printf("fat32 partition starts at byte %d\n", start)
root := ReadRootDir(os.Args[1])
fmt.Printf("%#v\n", root)
}
from go-diskfs.
The funny part is the above succeeds in reading it. This makes it much more difficult, as it is not clear where the failure is.
from go-diskfs.
Here’s an idea. Can you make a 5/1 image (the kind that fails) via the library, another via manual tools (and verify that works). I then can bite compare them.
from go-diskfs.
Will do
from go-diskfs.
Here are the images, same naming as before
go-diskfs
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# go run main.go /dev/sda
2020/01/29 22:27:49 Writing partition table to disk
2020/01/29 22:27:49 Creating cloud init directory structure
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# fdisk -lu /dev/sda
Disk /dev/sda: 5 GiB, 5368709120 bytes, 10485760 sectors
Disk model: Volume
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: dos
Disk identifier: 0x00000000
Device Boot Start End Sectors Size Id Type
/dev/sda1 8388608 10485759 2097152 1G 83 Linux
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# mount /dev/sda1 /mnt
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 5G 0 disk
└─sda1 8:1 0 1G 0 part /mnt
vda 252:0 0 160G 0 disk
└─vda1 252:1 0 160G 0 part /
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# blkid
/dev/vda1: UUID="b1438b9b-2cab-4065-a99a-08a96687f73c" TYPE="ext4" PARTUUID="7a0392d3-01"
/dev/sda1: LABEL_FATBOOT="config-2" UUID="4EF0-7AFF" TYPE="vfat"
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# umount /mnt
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# dd if=/dev/sda of=go-diskfs.img
10485760+0 records in
10485760+0 records out
5368709120 bytes (5.4 GB, 5.0 GiB) copied, 51.4466 s, 104 MB/s
manual
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# fdisk /dev/sda
Welcome to fdisk (util-linux 2.34).
Changes will remain in memory only, until you decide to write them.
Be careful before using the write command.
Device does not contain a recognized partition table.
Created a new DOS disklabel with disk identifier 0x6b97644a.
Command (m for help): n
Partition type
p primary (0 primary, 0 extended, 4 free)
e extended (container for logical partitions)
Select (default p):
Using default response p.
Partition number (1-4, default 1):
First sector (2048-10485759, default 2048): 8388608
Last sector, +/-sectors or +/-size{K,M,G,T,P} (8388608-10485759, default 10485759): 10485759
Created a new partition 1 of type 'Linux' and of size 1 GiB.
Command (m for help): p
Disk /dev/sda: 5 GiB, 5368709120 bytes, 10485760 sectors
Disk model: Volume
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: dos
Disk identifier: 0x6b97644a
Device Boot Start End Sectors Size Id Type
/dev/sda1 8388608 10485759 2097152 1G 83 Linux
Command (m for help): w
The partition table has been altered.
Calling ioctl() to re-read partition table.
Syncing disks.
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# mkfs.vfat -F 32 -s 2 /dev/sda1
mkfs.fat 4.1 (2017-01-24)
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# fatlabel /dev/sda1 config-2
fatlabel: warning - lowercase labels might not work properly with DOS or Windows
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# mount /dev/sda1 /mnt
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# mkdir -p /mnt/openstack/latest
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# ls -la /mnt/
total 6
drwxr-xr-x. 3 root root 1024 Jan 1 1970 .
dr-xr-xr-x. 18 root root 4096 Jan 29 14:43 ..
drwxr-xr-x. 3 root root 1024 Jan 29 22:33 openstack
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 5G 0 disk
└─sda1 8:1 0 1G 0 part /mnt
vda 252:0 0 160G 0 disk
└─vda1 252:1 0 160G 0 part /
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# blkid
/dev/vda1: UUID="b1438b9b-2cab-4065-a99a-08a96687f73c" TYPE="ext4" PARTUUID="7a0392d3-01"
/dev/sda1: LABEL_FATBOOT="config-2" LABEL="config-2" UUID="804C-E019" TYPE="vfat" PARTUUID="6b97644a-01"
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# umount /mnt
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# dd if=/dev/sda of=manual.img
10485760+0 records in
10485760+0 records out
5368709120 bytes (5.4 GB, 5.0 GiB) copied, 46.7378 s, 115 MB/s
from go-diskfs.
That definitely helped. I removed the partition from the image using dd
, and then tried to mount just the partition file. I got the same response, both on Linux and on Mac. This indicates that it isn't some odd partitioning thing, but something to do with the calculation of sizes for the filesystem in the partition itself, and makes it easier to work with.
FWIW:
$ dd skip=8388608 bs=512 if=./go-diskfs.img of=./go-diskfs-partition.img
$ dd skip=8388608 bs=512 if=./manual.img of=./manual-partition.img
And then just mount go-diskfs-partition.img
and manual-partition.img
.
from go-diskfs.
Awesome, glad you can see a difference.
from go-diskfs.
This is not going to be a fun one to debug, but I am cautiously optimistic.
from go-diskfs.
Let me know if you need me to generate any more images or try different things. I'm open to help in any way I can.
from go-diskfs.
Actually, I am curious why, when you ran it manually, you overrode to set the sectors per cluster to 2 with -s 2
?
from go-diskfs.
Umm I actually can't remember. I thought I saw somewhere in the go-diskfs code it setting it to 2 but I can't find that anymore... so maybe I just made it up lol
I can regenerate the manual image without setting -s 2
from go-diskfs.
Or set it to 8 maybe? There is no official spec for it, but go-diskfs defaults to using what most of the Linux utilities nominally use. A lot of this is unofficial "dirty art/magic" stuff.
from go-diskfs.
Thanks, that helped. I am seeing slight unexplained differences in the FAT tables.
from go-diskfs.
@rmb938 can you validate #50? My tests show it works.
from go-diskfs.
So the fix works but now the drive label is showing up as a file
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# go run main.go /dev/sda
2020/01/31 14:31:54 Writing partition table to disk
2020/01/31 14:31:54 Creating cloud init directory structure
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# mount /dev/sda1 /mnt
[root@fedora-s-4vcpu-8gb-sfo2-01 end_part]# ls -la /mnt
total 12
drwxr-xr-x. 3 root root 4096 Jan 1 1970 .
dr-xr-xr-x. 18 root root 4096 Jan 29 14:43 ..
-rwxr-xr-x. 1 root root 0 Jan 31 14:31 config-2
drwxr-xr-x. 3 root root 4096 Jan 31 14:31 openstack
from go-diskfs.
good to know. We will get that.
from go-diskfs.
That is an issue in the volume label creation, not in the size issue. I am going to merge that one in, then we can fix the other separately.
from go-diskfs.
from go-diskfs.
@deitch 👍 that fixed it. Thanks so much for all the help.
from go-diskfs.
Related Issues (20)
- Partitions not created HOT 15
- SectorSizeDefault not declared by package diskfs HOT 3
- Is 0o666 a typo? HOT 1
- Can't seem to read fat32 partition HOT 1
- Any plans for a release version? HOT 3
- Implement standard fs.FS interface and friends HOT 3
- Support status for solaris, aix, freebsd, illumos, netbsd, openbsd and plan9 HOT 3
- Incorrect number of partitions reported for the Ubuntu 22.04.02 Server ISO HOT 4
- corrupt GPT with examples/efi_create.go HOT 10
- Create ISO from existing image files with Rock Ridge enabled still errors on filenames longer than 30 characters HOT 11
- Support to remove files HOT 3
- squashfs: read target of symlinks HOT 5
- squashfs: caching of metadata and fragment blocks HOT 8
- Be able to compile to webassembly wasm HOT 3
- need help with finding resourses HOT 1
- Wrong Boot Information Table for ElTorito BIOS boot image larger than 64k HOT 4
- Wrong timestamps for special FAT-32 directory entries HOT 3
- Finalize error with a combination of RockRidge and ElTorito HOT 8
- Permissions for the filesystems supporting them? HOT 1
- Missing "FILE SERIAL NUMBER" when creating rockridge iso9660 files? HOT 2
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from go-diskfs.