From 0cc6757c579b9cbff64f4d8396b423892cff68f2 Mon Sep 17 00:00:00 2001 From: Gary Talent Date: Sat, 14 Sep 2024 20:47:54 -0500 Subject: [PATCH] [keel] Add manifest to pack output --- .gitignore | 1 + scripts/pkg-gba.py | 6 +++- src/olympic/keel/include/keel/media.hpp | 2 -- src/olympic/keel/include/keel/pack.hpp | 39 ++++++++++++++++++++++--- src/olympic/keel/src/media.cpp | 12 -------- src/olympic/keel/src/pack-applib.cpp | 17 ++++++++--- src/olympic/keel/src/pack.cpp | 36 ++++++++++++++++++----- 7 files changed, 82 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index 35f02221..3186e147 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ imgui.ini *.sav studio_state.json tags +*-manifest.json diff --git a/scripts/pkg-gba.py b/scripts/pkg-gba.py index f47c70f2..e3ee3692 100755 --- a/scripts/pkg-gba.py +++ b/scripts/pkg-gba.py @@ -28,8 +28,12 @@ project_name = sys.argv[2] bin = f'./build/{host_env}-{current_build}/bin/' project_bin = f'build/gba-release/bin/{project_name}.bin' project_gba = f'{project_name}.gba' +project_manifest = f'{project_name}-manifest.json' shutil.copyfile(project_bin, project_gba) subprocess.run([ - f'{bin}/{project_name}-pack', '-src', project_dir, '-rom-bin', project_gba]) + f'{bin}/{project_name}-pack', + '-src', project_dir, + '-rom-bin', project_gba, + '-manifest', project_manifest]) subprocess.run(['gbafix', project_gba]) diff --git a/src/olympic/keel/include/keel/media.hpp b/src/olympic/keel/include/keel/media.hpp index 1af4e491..29b94ad4 100644 --- a/src/olympic/keel/include/keel/media.hpp +++ b/src/olympic/keel/include/keel/media.hpp @@ -57,8 +57,6 @@ ox::Result uuidToPath(Context &ctx, ox::CRStringView uuid) noex ox::Result uuidToPath(Context &ctx, ox::UUID const&uuid) noexcept; -ox::Error performPackTransforms(Context &ctx, ox::Buffer &clawData) noexcept; - #ifndef OX_BARE_METAL namespace detail { diff --git a/src/olympic/keel/include/keel/pack.hpp b/src/olympic/keel/include/keel/pack.hpp index 635fc8ae..8d018208 100644 --- a/src/olympic/keel/include/keel/pack.hpp +++ b/src/olympic/keel/include/keel/pack.hpp @@ -12,6 +12,31 @@ namespace keel { +struct ManifestEntry { + static constexpr auto TypeName = "net.drinkingtea.keel.ManifestEntry"; + static constexpr auto TypeVersion = 1; + uint64_t inode{}; + bool preloaded{}; + ox::String type; +}; + +oxModelBegin(ManifestEntry) + oxModelField(inode) + oxModelField(preloaded) + oxModelField(type) +oxModelEnd() + +struct Manifest { + static constexpr auto TypeName = "net.drinkingtea.keel.Manifest"; + static constexpr auto TypeVersion = 1; + ox::HashMap files; +}; + +oxModelBegin(Manifest) + oxModelField(files) +oxModelEnd() + + class Context; struct GbaPlatSpec { @@ -113,6 +138,7 @@ ox::Error preloadObj( // transformations need to be done after the copy to the new FS is complete template ox::Error preloadDir( + Manifest &manifest, ox::TypeStore &ts, ox::FileSystem &romFs, ox::Preloader &pl, @@ -125,13 +151,14 @@ ox::Error preloadDir( oxRequire(stat, romFs.stat(filePath)); if (stat.fileType == ox::FileType::Directory) { auto const dir = ox::sfmt("{}{}/", path, name); - oxReturnError(preloadDir(ts, romFs, pl, dir)); + oxReturnError(preloadDir(manifest, ts, romFs, pl, dir)); } else { auto const err = preloadObj(ts, romFs, pl, filePath); if (err) { oxErrf("\033[31;1;1mCould not preload {}:\n\t{}\n", filePath, toStr(err)); return err; } + manifest.files[filePath].preloaded = true; } } return {}; @@ -162,11 +189,15 @@ ox::Error appendBinary(ox::Buffer &binBuff, ox::SpanView const&fsBuff, ox: } template -ox::Error preload(ox::TypeStore &ts, ox::FileSystem &src, ox::Preloader &pl) noexcept { +ox::Error preload( + Manifest &manifest, + ox::TypeStore &ts, + ox::FileSystem &src, + ox::Preloader &pl) noexcept { oxOut("Preloading\n"); - return detail::preloadDir(ts, src, pl, "/"); + return detail::preloadDir(manifest, ts, src, pl, "/"); } -ox::Error pack(keel::Context &ctx, ox::TypeStore &ts, ox::FileSystem &dest) noexcept; +ox::Error pack(Manifest &manifest, keel::Context &ctx, ox::TypeStore &ts, ox::FileSystem &dest) noexcept; } diff --git a/src/olympic/keel/src/media.cpp b/src/olympic/keel/src/media.cpp index ca3c857a..dfcad3e1 100644 --- a/src/olympic/keel/src/media.cpp +++ b/src/olympic/keel/src/media.cpp @@ -154,18 +154,6 @@ ox::Result uuidToPath(Context &ctx, ox::UUID const&uuid) noexce #endif } -ox::Error performPackTransforms(Context &ctx, ox::Buffer &clawData) noexcept { - oxRequireM(typeId, readAssetTypeId(clawData)); - for (auto const tr : packTransforms(ctx)) { - bool changed{}; - oxReturnError(tr(ctx, clawData, typeId).moveTo(changed)); - if (changed) { - oxReturnError(readAssetTypeId(clawData).moveTo(typeId)); - } - } - return {}; -} - ox::Error reloadAsset(keel::Context &ctx, ox::StringView assetId) noexcept { ox::UUIDStr uuidStr; if (beginsWith(assetId, "uuid://")) { diff --git a/src/olympic/keel/src/pack-applib.cpp b/src/olympic/keel/src/pack-applib.cpp index a830e98d..5ca13c88 100644 --- a/src/olympic/keel/src/pack-applib.cpp +++ b/src/olympic/keel/src/pack-applib.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -48,16 +49,21 @@ static ox::Error generateTypes(ox::TypeStore &ts) noexcept { return {}; } -static ox::Error pack(ox::StringView argSrc, ox::StringView argRomBin, ox::StringView projectDataDir) noexcept { +static ox::Error pack( + ox::StringView argSrc, + ox::StringView argRomBin, + ox::StringView argManifest, + ox::StringView projectDataDir) noexcept { ox::Buffer dstBuff(32 * ox::units::MB); oxReturnError(ox::FileSystem32::format(dstBuff.data(), dstBuff.size())); ox::FileSystem32 dst(dstBuff); oxRequire(ctx, keel::init(ox::make_unique(argSrc), "keel-pack")); keel::TypeStore ts(*ctx->rom, ox::sfmt("{}/type_descriptors", projectDataDir)); oxReturnError(generateTypes(ts)); - oxReturnError(keel::pack(*ctx, ts, dst)); + keel::Manifest manifest; + oxReturnError(keel::pack(manifest, *ctx, ts, dst)); oxRequireM(pl, keel::GbaPreloader::make()); - oxReturnError(preload(ts, dst, *pl)); + oxReturnError(preload(manifest, ts, dst, *pl)); oxReturnError(dst.resize()); // resize buffer oxRequire(dstSize, dst.size()); @@ -70,6 +76,8 @@ static ox::Error pack(ox::StringView argSrc, ox::StringView argRomBin, ox::Strin oxReturnError(appendBinary(romBuff, dstBuff, *pl)); oxOutf("Final ROM buff size: {} bytes\n", romBuff.size()); oxReturnError(writeFileBuff(argRomBin, romBuff)); + oxRequire(manifestJson, ox::writeOC(manifest)); + oxReturnError(writeFileBuff(argManifest, manifestJson)); return {}; } @@ -77,6 +85,7 @@ static ox::Error run(int argc, char const**argv, ox::StringView projectDataDir) ox::ClArgs const args(argc, argv); auto const argSrc = args.getString("src", ""); auto const argRomBin = args.getString("rom-bin", ""); + auto const argManifest = args.getString("manifest", ""); if (argSrc == "") { oxErr("\033[31;1;1merror:\033[0m must specify a source directory\n"); return OxError(1, "must specify a source directory"); @@ -85,7 +94,7 @@ static ox::Error run(int argc, char const**argv, ox::StringView projectDataDir) oxErr("\033[31;1;1merror:\033[0m must specify a path for ROM file\n"); return OxError(1, "must specify a path for preload file"); } - return pack(argSrc, argRomBin, projectDataDir); + return pack(argSrc, argRomBin, argManifest, projectDataDir); } namespace olympic { diff --git a/src/olympic/keel/src/pack.cpp b/src/olympic/keel/src/pack.cpp index 6f92eb6f..47eaac80 100644 --- a/src/olympic/keel/src/pack.cpp +++ b/src/olympic/keel/src/pack.cpp @@ -6,7 +6,6 @@ #include #include - #include namespace keel { @@ -85,7 +84,21 @@ static ox::Error transformFileAddressesObj( return {}; } +static ox::Error performPackTransforms(ManifestEntry &entry, Context &ctx, ox::Buffer &clawData) noexcept { + oxRequireM(typeId, readAssetTypeId(clawData)); + for (auto const tr : packTransforms(ctx)) { + bool changed{}; + oxReturnError(tr(ctx, clawData, typeId).moveTo(changed)); + if (changed) { + oxReturnError(readAssetTypeId(clawData).moveTo(typeId)); + } + } + entry.type = ox::String{typeId}; + return {}; +} + static ox::Error doTransformations( + Manifest &manifest, keel::Context &ctx, ox::TypeStore &ts, ox::FileSystem &dest, @@ -94,7 +107,7 @@ static ox::Error doTransformations( oxRequire(s, dest.stat(filePath)); // do transformations oxRequireM(buff, dest.read(s.inode)); - oxReturnError(keel::performPackTransforms(ctx, buff)); + oxReturnError(keel::performPackTransforms(manifest.files[filePath], ctx, buff)); // transform FileAddresses oxRequireM(obj, keel::readAsset(ts, buff)); oxOutf("transforming {}\n", filePath); @@ -108,6 +121,7 @@ static ox::Error doTransformations( // claw file transformations are broken out from copy because path to inode // transformations need to be done after the copy to the new FS is complete static ox::Error transformClaw( + Manifest &manifest, keel::Context &ctx, ox::TypeStore &ts, ox::FileSystem &dest, @@ -120,9 +134,9 @@ static ox::Error transformClaw( oxRequire(stat, dest.stat(filePath)); if (stat.fileType == ox::FileType::Directory) { auto const dir = ox::sfmt("{}{}/", path, name); - oxReturnError(transformClaw(ctx, ts, dest, dir)); + oxReturnError(transformClaw(manifest, ctx, ts, dest, dir)); } else { - auto const err = doTransformations(ctx, ts, dest, filePath); + auto const err = doTransformations(manifest, ctx, ts, dest, filePath); if (err) { oxErrf("\033[31;1;1mCould not do transformations for {}:\n\t{}\n", filePath, toStr(err)); return err; @@ -133,6 +147,7 @@ static ox::Error transformClaw( } static ox::Error copy( + Manifest &manifest, ox::FileSystem &src, ox::FileSystem &dest, ox::CRStringView path, @@ -149,7 +164,7 @@ static ox::Error copy( oxRequire(stat, src.stat(currentFile)); if (stat.fileType == ox::FileType::Directory) { oxReturnError(dest.mkdir(currentFile, true)); - oxReturnError(copy(src, dest, currentFile + '/', childLogPrefix)); + oxReturnError(copy(manifest, src, dest, currentFile + '/', childLogPrefix)); } else { // load file oxOutf("{}copying file: {}...", childLogPrefix, currentFile); @@ -161,15 +176,20 @@ static ox::Error copy( // write file to dest oxReturnError(dest.write(currentFile, buff)); status = "OK"; + oxRequire(stat, dest.stat(currentFile)); + manifest.files[currentFile] = { + .inode = stat.inode, + .type = ox::String{keel::readAssetTypeId(buff).or_value({})}, + }; } } return {}; } -ox::Error pack(keel::Context &ctx, ox::TypeStore &ts, ox::FileSystem &dest) noexcept { - oxReturnError(copy(*ctx.rom, dest, "/")); +ox::Error pack(Manifest &manifest, keel::Context &ctx, ox::TypeStore &ts, ox::FileSystem &dest) noexcept { + oxReturnError(copy(manifest, *ctx.rom, dest, "/")); oxOut("Doing transforms\n"); - oxReturnError(transformClaw(ctx, ts, dest, "/")); + oxReturnError(transformClaw(manifest, ctx, ts, dest, "/")); return {}; }