[keel] Split out Nostalgia Foundation and Pack lib into Keel

This commit is contained in:
2023-03-24 21:20:55 -05:00
parent 4a95a79926
commit 7beb3cc6fc
50 changed files with 185 additions and 206 deletions

33
src/keel/CMakeLists.txt Normal file
View File

@ -0,0 +1,33 @@
add_library(
Keel
asset.cpp
media.cpp
module.cpp
pack.cpp
typeconv.cpp
)
target_link_libraries(
Keel PUBLIC
OxClaw
OxEvent
OxFS
OxModel
OxPreloader
OxStd
)
install(
FILES
assetmanager.hpp
context.hpp
keel.hpp
asset.hpp
media.hpp
module.hpp
pack.hpp
typeconv.hpp
DESTINATION
include/keel
)

46
src/keel/asset.cpp Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#include "asset.hpp"
namespace keel {
ox::Result<ox::UUID> readUuidHeader(const ox::Buffer &buff) noexcept {
return readUuidHeader(buff.data(), buff.size());
}
ox::Result<ox::UUID> readUuidHeader(const char *buff, std::size_t buffLen) noexcept {
if (buffLen < N1HdrSz) {
return OxError(1, "Insufficient data to contain complete Nostalgia header");
}
ox::StringView n1Hdr = "N1;";
if (n1Hdr == buff) {
return OxError(2, "No Nostalgia asset header data");
}
return ox::UUID::fromString(ox::StringView(buff + n1Hdr.bytes(), 36));
}
ox::Result<ox::ModelObject> readAsset(ox::TypeStore *ts, const ox::Buffer &buff) noexcept {
std::size_t offset = 0;
if (!readUuidHeader(buff).error) {
offset = N1HdrSz;
}
return ox::readClaw(ts, buff.data() + offset, buff.size() - offset);
}
ox::Result<AssetHdr> readAssetHeader(const char *buff, std::size_t buffLen) noexcept {
AssetHdr out;
const auto err = readUuidHeader(buff, buffLen).moveTo(&out.uuid);
const auto offset = err ? 0 : N1HdrSz;
buff = buff + offset;
buffLen = buffLen - offset;
oxReturnError(ox::readClawHeader(buff, buffLen).moveTo(&out.clawHdr));
return out;
}
ox::Result<AssetHdr> readAssetHeader(const ox::Buffer &buff) noexcept {
return readAssetHeader(buff.data(), buff.size());
}
}

46
src/keel/asset.hpp Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#pragma once
#include <ox/std/defines.hpp>
#include <ox/claw/claw.hpp>
#include <ox/fs/fs.hpp>
namespace keel {
constexpr auto N1HdrSz = 40;
ox::Result<ox::UUID> readUuidHeader(const ox::Buffer &buff) noexcept;
ox::Result<ox::UUID> readUuidHeader(const char *buff, std::size_t buffLen) noexcept;
ox::Error writeUuidHeader(ox::Writer_c auto *writer, const ox::UUID &uuid) noexcept {
const auto hdr = ox::sfmt<ox::BString<N1HdrSz>>("N1;{};", uuid.toString());
return write(writer, hdr);
}
template<typename T>
ox::Result<T> readAsset(const ox::Buffer &buff) noexcept {
std::size_t offset = 0;
const auto err = readUuidHeader(buff).error;
if (!err) {
offset = N1HdrSz; // the size of N1 headers
}
return ox::readClaw<T>(buff.data() + offset, buff.size() - offset);
}
ox::Result<ox::ModelObject> readAsset(ox::TypeStore *ts, const ox::Buffer &buff) noexcept;
struct AssetHdr {
ox::UUID uuid;
ox::ClawHeader clawHdr;
};
ox::Result<AssetHdr> readAssetHeader(const char *buff, std::size_t buffLen) noexcept;
ox::Result<AssetHdr> readAssetHeader(const ox::Buffer &buff) noexcept;
}

302
src/keel/assetmanager.hpp Normal file
View File

@ -0,0 +1,302 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#pragma once
#include <ox/event/signal.hpp>
#include <ox/fs/fs.hpp>
#include <ox/model/typenamecatcher.hpp>
#include <ox/std/hashmap.hpp>
#include <ox/std/utility.hpp>
namespace keel {
class AssetManager;
template<typename T>
class AssetRef;
#ifndef OX_BARE_METAL
template<typename T>
class AssetContainer {
friend AssetManager;
friend AssetRef<T>;
protected:
ox::Signal<ox::Error()> updated;
private:
T m_obj;
mutable int m_references = 0;
public:
template<class... Args>
explicit constexpr AssetContainer(Args&&... args): m_obj(ox::forward<Args>(args)...) {
}
AssetContainer(AssetContainer&) = delete;
AssetContainer(AssetContainer&&) = delete;
AssetContainer& operator=(AssetContainer&) = delete;
AssetContainer& operator=(AssetContainer&&) = delete;
[[nodiscard]]
constexpr T *get() noexcept {
return &m_obj;
}
[[nodiscard]]
constexpr const T *get() const noexcept {
return &m_obj;
}
constexpr void set(T &&val) {
m_obj = std::move(val);
}
constexpr void set(const T &val) {
m_obj = val;
}
protected:
constexpr void incRefs() const noexcept {
++m_references;
}
constexpr void decRefs() const noexcept {
--m_references;
}
[[nodiscard]]
constexpr int references() const noexcept {
return m_references;
}
};
template<typename T>
class AssetRef: public ox::SignalHandler {
private:
const AssetContainer<T> *m_ctr = nullptr;
public:
ox::Signal<ox::Error()> updated;
explicit constexpr AssetRef(const AssetContainer<T> *c = nullptr) noexcept;
constexpr AssetRef(const AssetRef &h) noexcept;
constexpr AssetRef(AssetRef &&h) noexcept;
~AssetRef() noexcept override {
if (m_ctr) {
m_ctr->decRefs();
}
}
[[nodiscard]]
constexpr const T *get() const noexcept {
if (m_ctr) {
return m_ctr->get();
}
return nullptr;
}
constexpr const T &operator*() const & noexcept {
return *m_ctr->get();
}
constexpr const T &&operator*() const && noexcept {
return *m_ctr->get();
}
constexpr const T *operator->() const noexcept {
return m_ctr->get();
}
AssetRef &operator=(const AssetRef &h) noexcept {
if (this == &h) {
return *this;
}
if (m_ctr) {
m_ctr->decRefs();
oxIgnoreError(m_ctr->updated.disconnectObject(this));
}
m_ctr = h.m_ctr;
m_ctr->updated.connect(&updated, &ox::Signal<ox::Error()>::emitCheckError);
if (m_ctr) {
m_ctr->incRefs();
}
return *this;
}
AssetRef &operator=(AssetRef &&h) noexcept {
if (this == &h) {
return *this;
}
if (m_ctr) {
m_ctr->decRefs();
oxIgnoreError(m_ctr->updated.disconnectObject(this));
}
m_ctr = h.m_ctr;
m_ctr->updated.connect(this, &AssetRef::emitUpdated);
h.m_ctr = nullptr;
return *this;
}
explicit constexpr operator bool() const noexcept {
return m_ctr;
}
private:
constexpr ox::Error emitUpdated() const noexcept {
updated.emit();
return OxError(0);
}
};
template<typename T>
constexpr AssetRef<T>::AssetRef(const AssetContainer<T> *c) noexcept: m_ctr(c) {
if (c) {
c->updated.connect(this, &AssetRef::emitUpdated);
}
}
template<typename T>
constexpr AssetRef<T>::AssetRef(const AssetRef &h) noexcept {
m_ctr = h.m_ctr;
if (m_ctr) {
m_ctr->updated.connect(this, &AssetRef::emitUpdated);
m_ctr->incRefs();
}
}
template<typename T>
constexpr AssetRef<T>::AssetRef(AssetRef &&h) noexcept {
m_ctr = h.m_ctr;
m_ctr->updated.connect(this, &AssetRef::emitUpdated);
h.m_ctr = nullptr;
}
class AssetManager {
private:
class AssetTypeManagerBase {
public:
virtual ~AssetTypeManagerBase() = default;
virtual void gc() noexcept = 0;
};
template<typename T>
class AssetTypeManager: public AssetTypeManagerBase {
private:
ox::HashMap<ox::String, ox::UniquePtr<AssetContainer<T>>> m_cache;
public:
ox::Result<AssetRef<T>> getAsset(const ox::String &assetId) const noexcept {
auto out = m_cache.at(assetId);
oxReturnError(out);
return AssetRef<T>(out.value->get());
}
ox::Result<AssetRef<T>> setAsset(const ox::String &assetId, const T &obj) noexcept {
auto &p = m_cache[assetId];
if (!p) {
p = ox::make_unique<AssetContainer<T>>(obj);
} else {
p->set(obj);
p->updated.emit();
}
return AssetRef<T>(p.get());
}
ox::Result<AssetRef<T>> setAsset(const ox::String &assetId, T &&obj) noexcept {
auto &p = m_cache[assetId];
if (!p) {
p = ox::make_unique<AssetContainer<T>>(obj);
} else {
p->set(std::move(obj));
p->updated.emit();
}
return AssetRef<T>(p.get());
}
void gc() noexcept final {
for (const auto &ack : m_cache.keys()) {
auto &ac = m_cache[ack];
if (!ac->references()) {
m_cache.erase(ack);
}
}
}
};
ox::HashMap<ox::String, ox::UniquePtr<AssetTypeManagerBase>> m_assetTypeManagers;
template<typename T>
AssetTypeManager<T> *getTypeManager() noexcept {
constexpr auto typeName = ox::requireModelTypeName<T>();
static_assert(ox_strcmp(typeName, "") != 0, "Types must have TypeName to use AssetManager");
auto &am = m_assetTypeManagers[typeName];
if (!am) {
am = ox::make_unique<AssetTypeManager<T>>();
}
return dynamic_cast<AssetTypeManager<T>*>(am.get());
}
public:
template<typename T>
ox::Result<AssetRef<T>> getAsset(const ox::String &assetId) noexcept {
auto m = getTypeManager<T>();
return m->getAsset(assetId);
}
template<typename T>
ox::Result<AssetRef<T>> setAsset(const ox::String &assetId, const T &obj) noexcept {
auto m = getTypeManager<T>();
return m->setAsset(assetId, obj);
}
void gc() noexcept {
for (const auto &amk : m_assetTypeManagers.keys()) {
auto &am = m_assetTypeManagers[amk];
am->gc();
}
}
};
#else
template<typename T>
class AssetRef {
private:
const T *m_obj = nullptr;
public:
constexpr AssetRef() noexcept = default;
explicit constexpr AssetRef(const T *obj) noexcept: m_obj(obj) {
}
constexpr const T *get() const noexcept {
return m_obj;
}
constexpr const T &operator*() const & noexcept {
return *m_obj;
}
constexpr const T &&operator*() const && noexcept {
return *m_obj;
}
constexpr const T *operator->() const noexcept {
return m_obj;
}
explicit constexpr operator bool() const noexcept {
return m_obj;
}
};
#endif
}

40
src/keel/context.hpp Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#pragma once
#include <ox/fs/fs.hpp>
#include <ox/std/memory.hpp>
#include <ox/std/stringview.hpp>
#include "assetmanager.hpp"
namespace keel {
class Context;
using PackTransform = ox::Error(*)(Context*, ox::Buffer *clawData);
class Context {
public:
ox::UPtr<ox::FileSystem> rom;
ox::StringView appName = "Nostalgia Foundation App";
#ifndef OX_BARE_METAL
AssetManager assetManager;
ox::HashMap<ox::String, ox::UUID> pathToUuid;
ox::HashMap<ox::UUIDStr, ox::String> uuidToPath;
ox::Vector<const class BaseConverter*> converters;
ox::Vector<PackTransform> packTransforms;
#else
std::size_t preloadSectionOffset = 0;
#endif
constexpr Context() noexcept = default;
Context(const Context&) noexcept = delete;
Context(Context&&) noexcept = delete;
Context &operator=(const Context&) noexcept = delete;
Context &operator=(Context&&) noexcept = delete;
constexpr virtual ~Context() noexcept = default;
};
}

35
src/keel/keel.hpp Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#pragma once
#include <ox/std/memory.hpp>
#include "context.hpp"
#include "media.hpp"
#include "module.hpp"
#include "pack.hpp"
namespace keel {
template<typename Ctx = keel::Context>
ox::Result<ox::UPtr<Ctx>> init(ox::UPtr<ox::FileSystem> &&fs, ox::CRStringView appName) noexcept {
auto ctx = ox::make_unique<Ctx>();
ctx->appName = appName;
oxIgnoreError(setRomFs(ctx.get(), std::move(fs)));
const auto &mods = modules();
for (auto &mod : mods) {
// register type converters
for (auto c : mod->converters()) {
ctx->converters.emplace_back(c);
}
// register pack transforms
for (auto c : mod->packTransforms()) {
ctx->packTransforms.emplace_back(c);
}
}
return ctx;
}
}

160
src/keel/media.cpp Normal file
View File

@ -0,0 +1,160 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#include "media.hpp"
#ifndef OX_BARE_METAL
#include <fstream>
#include <ox/std/trace.hpp>
#include "media.hpp"
namespace keel {
ox::Result<char*> loadRom(ox::CRStringView path) noexcept {
std::ifstream file(std::string(toStdStringView(path)), std::ios::binary | std::ios::ate);
if (!file.good()) {
oxErrorf("Could not find ROM file: {}", path);
return OxError(1, "Could not find ROM file");
}
try {
const auto size = file.tellg();
file.seekg(0, std::ios::beg);
auto buff = new char[static_cast<std::size_t>(size)];
file.read(buff, size);
return buff;
} catch (const std::ios_base::failure &e) {
oxErrorf("Could not read ROM file: {}", e.what());
return OxError(2, "Could not read ROM file");
}
}
void unloadRom(char *rom) noexcept {
ox::safeDelete(rom);
}
ox::Result<void*> findPreloadSection() noexcept {
return OxError(1, "findPreloadSection is unsupported on this platform");
}
static void clearUuidMap(Context *ctx) noexcept {
ctx->uuidToPath.clear();
ctx->pathToUuid.clear();
}
void createUuidMapping(Context *ctx, const ox::String &filePath, const ox::UUID &uuid) noexcept {
ctx->pathToUuid[filePath] = uuid;
ctx->uuidToPath[uuid.toString()] = filePath;
}
static ox::Error buildUuidMap(Context *ctx, ox::CRStringView path) noexcept {
oxRequire(files, ctx->rom->ls(path));
for (const auto &f : files) {
oxRequire(filePath, ox::join("/", ox::Array<ox::StringView, 2>{path, f}));
oxRequire(stat, ctx->rom->stat(filePath));
if (stat.fileType == ox::FileType::NormalFile) {
oxRequire(data, ctx->rom->read(filePath));
const auto [hdr, err] = readAssetHeader(data);
if (!err) {
createUuidMapping(ctx, filePath, hdr.uuid);
}
} else if (stat.fileType == ox::FileType::Directory) {
if (!beginsWith(f, ".")) {
oxReturnError(buildUuidMap(ctx, filePath));
}
}
}
return {};
}
ox::Error buildUuidMap(Context *ctx) noexcept {
if (!ctx->rom) {
return OxError(1, "No ROM FS");
}
return buildUuidMap(ctx, "");
}
}
#else
#include "context.hpp"
#define MEM_ROM reinterpret_cast<char*>(0x0800'0000)
namespace keel {
static void clearUuidMap(Context*) noexcept {
}
ox::Error buildUuidMap(Context*) noexcept {
return {};
}
ox::Result<char*> loadRom(ox::CRStringView) noexcept {
// put the header in the wrong order to prevent mistaking this code for the
// media section
constexpr auto headerP2 = "HEADER__________";
constexpr auto headerP1 = "NOSTALGIA_MEDIA_";
constexpr auto headerP1Len = ox_strlen(headerP2);
constexpr auto headerP2Len = ox_strlen(headerP1);
constexpr auto headerLen = headerP1Len + headerP2Len;
for (auto current = MEM_ROM; current < reinterpret_cast<char*>(0x0a000000); current += headerLen) {
if (ox_memcmp(current, headerP1, headerP1Len) == 0 &&
ox_memcmp(current + headerP1Len, headerP2, headerP2Len) == 0) {
return current + headerLen;
}
}
return OxError(1);
}
void unloadRom(char*) noexcept {
}
ox::Result<std::size_t> getPreloadAddr(keel::Context *ctx, ox::CRStringView path) noexcept {
oxRequire(stat, ctx->rom->stat(path));
oxRequire(buff, static_cast<ox::MemFS*>(ctx->rom.get())->directAccess(path));
PreloadPtr p;
oxReturnError(ox::readMC(buff, stat.size, &p));
return p.preloadAddr + ctx->preloadSectionOffset;
}
ox::Result<std::size_t> getPreloadAddr(keel::Context *ctx, const ox::FileAddress &file) noexcept {
oxRequire(stat, ctx->rom->stat(file));
oxRequire(buff, static_cast<ox::MemFS*>(ctx->rom.get())->directAccess(file));
PreloadPtr p;
oxReturnError(ox::readMC(buff, stat.size, &p));
return p.preloadAddr + ctx->preloadSectionOffset;
}
}
#endif
namespace keel {
ox::Error setRomFs(Context *ctx, ox::UPtr<ox::FileSystem> fs) noexcept {
ctx->rom = std::move(fs);
clearUuidMap(ctx);
return buildUuidMap(ctx);
}
ox::Result<ox::UniquePtr<ox::FileSystem>> loadRomFs(ox::CRStringView path) noexcept {
const auto lastDot = ox_lastIndexOf(path, '.');
const auto fsExt = lastDot != -1 ? path.substr(static_cast<std::size_t>(lastDot)) : "";
if (ox_strcmp(fsExt, ".oxfs") == 0) {
oxRequire(rom, loadRom(path));
return {ox::make_unique<ox::FileSystem32>(rom, 32 * ox::units::MB, unloadRom)};
} else {
#ifdef OX_HAS_PASSTHROUGHFS
return {ox::make_unique<ox::PassThroughFS>(path)};
#else
return OxError(2);
#endif
}
}
}

146
src/keel/media.hpp Normal file
View File

@ -0,0 +1,146 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#pragma once
#include <ox/std/defines.hpp>
#include <ox/claw/claw.hpp>
#include <ox/fs/fs.hpp>
#include <ox/model/metadata.hpp>
#include "asset.hpp"
#include "context.hpp"
#include "typeconv.hpp"
namespace keel {
// Pointer to preloaded data that can be stored in FS in place of the actual
// data.
struct PreloadPtr {
static constexpr auto TypeName = "net.drinkingtea.ox.PreloadPtr";
static constexpr auto TypeVersion = 1;
uint32_t preloadAddr = 0;
};
oxModelBegin(PreloadPtr)
oxModelField(preloadAddr)
oxModelEnd()
ox::Result<std::size_t> getPreloadAddr(keel::Context *ctx, const ox::FileAddress &file) noexcept;
ox::Result<std::size_t> getPreloadAddr(keel::Context *ctx, ox::CRStringView file) noexcept;
#ifndef OX_BARE_METAL
template<typename T>
ox::Result<keel::AssetRef<T>> readObjFile(
keel::Context *ctx,
ox::StringView assetId,
bool forceLoad) noexcept {
constexpr auto readConvert = [](Context *ctx, const ox::Buffer &buff) -> ox::Result<T> {
auto [obj, err] = readAsset<T>(buff);
if (err) {
if (err != ox::Error_ClawTypeVersionMismatch && err != ox::Error_ClawTypeMismatch) {
return err;
}
oxReturnError(convert<T>(ctx, buff, &obj));
}
return std::move(obj);
};
ox::StringView path;
ox::UUIDStr uuidStr;
if (beginsWith(assetId, "uuid://")) {
assetId = assetId.substr(7);
path = ctx->uuidToPath[assetId];
} else {
path = assetId;
uuidStr = ctx->pathToUuid[path].toString();
assetId = uuidStr;
}
if (forceLoad) {
oxRequire(buff, ctx->rom->read(path));
oxRequire(obj, readConvert(ctx, buff));
oxRequire(cached, ctx->assetManager.setAsset(assetId, obj));
return cached;
} else {
auto [cached, err] = ctx->assetManager.getAsset<T>(assetId);
if (err) {
oxRequire(buff, ctx->rom->read(path));
oxRequire(obj, readConvert(ctx, buff));
oxReturnError(ctx->assetManager.setAsset(assetId, obj).moveTo(&cached));
}
return cached;
}
}
#else
template<typename T>
ox::Result<keel::AssetRef<T>> readObjNoCache(
keel::Context *ctx,
ox::CRStringView assetId) noexcept {
if constexpr(ox::preloadable<T>::value) {
oxRequire(addr, getPreloadAddr(ctx, assetId));
return keel::AssetRef<T>(reinterpret_cast<const T*>(addr));
} else {
return OxError(1);
}
}
#endif
void createUuidMapping(Context *ctx, const ox::String &filePath, const ox::UUID &uuid) noexcept;
ox::Error buildUuidMap(Context *ctx) noexcept;
template<typename T>
ox::Result<keel::AssetRef<T>> readObj(
[[maybe_unused]] keel::Context *ctx,
[[maybe_unused]] ox::CRStringView assetId,
[[maybe_unused]] bool forceLoad = false) noexcept {
#ifndef OX_BARE_METAL
return readObjFile<T>(ctx, assetId, forceLoad);
#else
return readObjNoCache<T>(ctx, assetId);
#endif
}
template<typename T>
ox::Result<keel::AssetRef<T>> readObj(
keel::Context *ctx,
const ox::FileAddress &file,
[[maybe_unused]] bool forceLoad = false) noexcept {
#ifndef OX_BARE_METAL
oxRequire(assetId, file.getPath());
return readObj<T>(ctx, ox::StringView(assetId), forceLoad);
#else
if constexpr(ox::preloadable<T>::value) {
oxRequire(addr, getPreloadAddr(ctx, file));
return keel::AssetRef<T>(reinterpret_cast<const T*>(addr));
} else {
return OxError(1);
}
#endif
}
template<typename T>
ox::Error writeObj(
keel::Context *ctx,
const ox::FileAddress &file,
const T &obj,
ox::ClawFormat fmt = ox::ClawFormat::Metal) noexcept {
oxRequire(objBuff, ox::writeClaw(&obj, fmt));
return ctx->rom->write(file, objBuff.data(), objBuff.size());
}
ox::Error setRomFs(Context *ctx, ox::UPtr<ox::FileSystem> fs) noexcept;
ox::Result<ox::UniquePtr<ox::FileSystem>> loadRomFs(ox::CRStringView path) noexcept;
ox::Result<char*> loadRom(ox::CRStringView assetId = "") noexcept;
void unloadRom(char*) noexcept;
}

33
src/keel/module.cpp Normal file
View File

@ -0,0 +1,33 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#include "module.hpp"
namespace keel {
static ox::Vector<const Module*> mods;
void registerModule(const Module *mod) noexcept {
mods.emplace_back(mod);
}
[[nodiscard]]
const ox::Vector<const Module*> &modules() noexcept {
return mods;
}
ox::Vector<TypeDescGenerator> Module::types() const noexcept {
return {};
}
ox::Vector<const keel::BaseConverter*> Module::converters() const noexcept {
return {};
}
ox::Vector<PackTransform> Module::packTransforms() const noexcept {
return {};
}
}

43
src/keel/module.hpp Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#pragma once
#include <ox/std/vector.hpp>
#include <ox/model/descwrite.hpp>
#include "typeconv.hpp"
namespace keel {
using TypeDescGenerator = ox::Error(*)(ox::TypeStore*);
template<typename T>
ox::Error generateTypeDesc(ox::TypeStore *ts) noexcept {
return ox::buildTypeDef<T>(ts).error;
}
class Module {
public:
constexpr Module() noexcept = default;
Module(const Module&) noexcept = delete;
Module(Module&&) noexcept = delete;
Module &operator=(const Module&) noexcept = delete;
Module &operator=(Module&&) noexcept = delete;
constexpr virtual ~Module() noexcept = default;
[[nodiscard]]
virtual ox::Vector<TypeDescGenerator> types() const noexcept;
[[nodiscard]]
virtual ox::Vector<const keel::BaseConverter*> converters() const noexcept;
[[nodiscard]]
virtual ox::Vector<PackTransform> packTransforms() const noexcept;
};
void registerModule(const Module *mod) noexcept;
[[nodiscard]]
const ox::Vector<const keel::Module*> &modules() noexcept;
}

247
src/keel/pack.cpp Normal file
View File

@ -0,0 +1,247 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#ifndef OX_BARE_METAL
#include <ox/claw/read.hpp>
#include <ox/fs/fs.hpp>
#include <ox/model/descwrite.hpp>
#include <ox/model/modelvalue.hpp>
#include <keel/media.hpp>
#include "pack.hpp"
namespace keel {
static ox::Error pathToInode(keel::Context *ctx, ox::FileSystem *dest, ox::ModelObject *obj) noexcept {
auto &o = *obj;
auto type = static_cast<ox::FileAddressType>(o["type"].get<int8_t>());
auto &data = o["data"].get<ox::ModelUnion>();
ox::String path;
switch (type) {
case ox::FileAddressType::Path:
path = data["path"].get<ox::String>();
break;
case ox::FileAddressType::ConstPath:
path = data["constPath"].get<ox::String>();
break;
case ox::FileAddressType::Inode:
case ox::FileAddressType::None:
return {};
}
if (beginsWith(path, "uuid://")) {
const auto uuid = ox::StringView(path).substr(7);
path = ctx->uuidToPath[uuid];
}
oxRequire(s, dest->stat(path));
oxReturnError(o["type"].set(static_cast<int8_t>(ox::FileAddressType::Inode)));
return data.set(2, s.inode);
}
static ox::Error transformFileAddressesObj(keel::Context *ctx, ox::FileSystem *dest, ox::ModelObject *obj) noexcept;
static ox::Error transformFileAddressesVec(keel::Context *ctx, ox::FileSystem *dest, ox::ModelValueVector *v) noexcept;
static ox::Error transformFileAddresses(keel::Context *ctx, ox::FileSystem *dest, ox::ModelValue *v) noexcept {
if (v->type() == ox::ModelValue::Type::Object) {
auto &obj = v->get<ox::ModelObject>();
return transformFileAddressesObj(ctx, dest, &obj);
} else if (v->type() == ox::ModelValue::Type::Vector) {
auto &vec = v->get<ox::ModelValueVector>();
return transformFileAddressesVec(ctx, dest, &vec);
}
return {};
}
static ox::Error transformFileAddressesVec(keel::Context *ctx, ox::FileSystem *dest, ox::ModelValueVector *v) noexcept {
for (auto &f : *v) {
oxReturnError(transformFileAddresses(ctx, dest, &f));
}
return {};
}
/**
* Convert path references in Claw data to inodes to save space
* @return error
*/
static ox::Error transformFileAddressesObj(keel::Context *ctx, ox::FileSystem *dest, ox::ModelObject *obj) noexcept {
if (obj->typeName() == "net.drinkingtea.ox.FileAddress" && obj->typeVersion() == 1) {
return pathToInode(ctx, dest, obj);
}
for (auto &f : *obj) {
auto &v = f->value;
oxReturnError(transformFileAddresses(ctx, dest, &v));
}
return {};
}
static ox::Error doTransformations(keel::Context *ctx, ox::TypeStore *ts, ox::FileSystem *dest, ox::CRStringView filePath) noexcept {
// load file
oxRequire(s, dest->stat(filePath));
// do transformations
oxRequireM(buff, dest->read(s.inode));
for (auto tr : ctx->packTransforms) {
oxReturnError(tr(ctx, &buff));
}
// transform FileAddresses
oxRequireM(obj, keel::readAsset(ts, buff));
oxReturnError(transformFileAddressesObj(ctx, dest, &obj));
oxReturnError(ox::writeClaw(&obj).moveTo(&buff));
// write file to dest
oxReturnError(dest->write(s.inode, buff.data(), buff.size()));
return {};
}
// claw file transformations are broken out from copy because path to inode
// transformations need to be done after the copy to the new FS is complete
static ox::Error transformClaw(keel::Context *ctx, ox::TypeStore *ts, ox::FileSystem *dest, ox::CRStringView path) noexcept {
// copy
oxTracef("pack::transformClaw", "path: {}", path);
oxRequire(fileList, dest->ls(path));
for (const auto &name : fileList) {
const auto filePath = ox::sfmt("{}{}", path, name);
oxRequire(stat, dest->stat(filePath));
if (stat.fileType == ox::FileType::Directory) {
const auto dir = ox::sfmt("{}{}/", path, name);
oxReturnError(transformClaw(ctx, ts, dest, dir));
} else {
oxReturnError(doTransformations(ctx, ts, dest, filePath));
}
}
return {};
}
static ox::Error verifyFile(ox::FileSystem *fs, ox::CRStringView path, const ox::Buffer &expected) noexcept {
ox::Buffer buff(expected.size());
oxReturnError(fs->read(path, buff.data(), buff.size()));
return OxError(buff == expected ? 0 : 1);
}
struct VerificationPair {
ox::String path;
ox::Buffer buff;
VerificationPair(ox::String &&pPath, ox::Buffer &&pBuff) noexcept:
path(std::forward<ox::String>(pPath)),
buff(std::forward<ox::Buffer>(pBuff)) {
}
};
static ox::Error copy(ox::FileSystem *src, ox::FileSystem *dest, ox::CRStringView path, ox::Vector<VerificationPair> *verificationPairs) noexcept {
oxOutf("copying directory: {}\n", path);
// copy
oxRequire(fileList, src->ls(path));
for (const auto &name : fileList) {
auto currentFile = ox::sfmt("{}{}", path, name);
if (beginsWith(name, ".")) {
continue;
}
oxOutf("reading {}\n", currentFile);
oxRequire(stat, src->stat(currentFile));
if (stat.fileType == ox::FileType::Directory) {
oxReturnError(dest->mkdir(currentFile, true));
oxReturnError(copy(src, dest, currentFile + '/', verificationPairs));
} else {
// load file
oxRequireM(buff, src->read(currentFile));
// write file to dest
oxOutf("writing {}\n", currentFile);
oxReturnError(dest->write(currentFile, buff.data(), buff.size()));
oxReturnError(verifyFile(dest, currentFile, buff));
verificationPairs->emplace_back(std::move(currentFile), std::move(buff));
}
}
return {};
}
static ox::Error copyFS(ox::FileSystem *src, ox::FileSystem *dest) noexcept {
ox::Vector<VerificationPair> verificationPairs;
oxReturnError(copy(src, dest, "/", &verificationPairs));
// verify all at once in addition to right after the files are written
oxOutf("Verifying completed destination\n");
for (const auto &v : verificationPairs) {
oxReturnError(verifyFile(dest, v.path, v.buff));
}
return {};
}
// transformations need to be done after the copy to the new FS is complete
static ox::Error preloadObj(
ox::TypeStore *ts, ox::FileSystem *romFs,
GbaPreloader *pl, ox::CRStringView path) noexcept {
// load file
oxRequireM(buff, romFs->read(path));
oxRequireM(obj, keel::readAsset(ts, buff));
if (obj.type()->preloadable) {
oxOutf("preloading {}\n", path);
// preload
oxRequire(a, pl->startAlloc(ox::sizeOf<GbaPlatSpec>(&obj)));
const auto err = ox::preload<GbaPlatSpec, decltype(obj)>(pl, &obj);
oxReturnError(pl->endAlloc());
oxReturnError(err);
const keel::PreloadPtr p{.preloadAddr = static_cast<uint32_t>(a)};
oxReturnError(ox::writeMC(&p).moveTo(&buff));
} else {
// strip the Claw header (it is not needed after preloading) and write back out to dest fs
oxReturnError(ox::writeMC(&obj).moveTo(&buff));
}
oxReturnError(romFs->write(path, buff.data(), buff.size()));
return {};
}
// claw file transformations are broken out because path to inode
// transformations need to be done after the copy to the new FS is complete
static ox::Error preloadDir(ox::TypeStore *ts, ox::FileSystem *romFs, GbaPreloader *pl, ox::CRStringView path) noexcept {
// copy
oxTracef("pack::preload", "path: {}", path);
oxRequire(fileList, romFs->ls(path));
for (const auto &name : fileList) {
const auto filePath = ox::sfmt("{}{}", path, name);
oxRequire(stat, romFs->stat(filePath));
if (stat.fileType == ox::FileType::Directory) {
const auto dir = ox::sfmt("{}{}/", path, name);
oxReturnError(preloadDir(ts, romFs, pl, dir));
} else {
oxReturnError(preloadObj(ts, romFs, pl, filePath));
}
}
return {};
}
static ox::Error padbin(ox::BufferWriter *w, unsigned factor) noexcept {
return w->write(nullptr, factor - w->buff().size() % factor);
}
ox::Error appendBinary(ox::Buffer *binBuff, ox::Buffer *fsBuff, GbaPreloader *pl) noexcept {
constexpr ox::StringView mediaHdr = "NOSTALGIA_MEDIA_HEADER__________";
constexpr ox::StringView preloadHdr = "NOSTALGIA_PRELOAD_HEADER________";
constexpr auto hdrSize = 32u;
static_assert(mediaHdr.bytes() == hdrSize);
static_assert(preloadHdr.bytes() == hdrSize);
ox::BufferWriter w(binBuff);
oxReturnError(padbin(&w, hdrSize));
oxReturnError(w.write(mediaHdr.data(), mediaHdr.bytes()));
oxReturnError(w.write(fsBuff->data(), fsBuff->size()));
oxReturnError(padbin(&w, hdrSize));
oxReturnError(w.write(preloadHdr.data(), preloadHdr.bytes()));
const auto &plBuff = pl->buff();
oxReturnError(pl->offsetPtrs(binBuff->size()));
oxReturnError(w.write(plBuff.data(), plBuff.size()));
return {};
}
ox::Error pack(keel::Context *ctx, ox::TypeStore *ts, ox::FileSystem *dest) noexcept {
oxReturnError(copyFS(ctx->rom.get(), dest));
oxOut("Doing transforms\n");
oxReturnError(transformClaw(ctx, ts, dest, "/"));
return {};
}
ox::Error preload(ox::TypeStore *ts, ox::FileSystem *src, GbaPreloader *pl) noexcept {
oxOut("Preloading\n");
return preloadDir(ts, src, pl, "/");
}
}
#endif

87
src/keel/pack.hpp Normal file
View File

@ -0,0 +1,87 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#pragma once
#include <ox/fs/fs.hpp>
#include <ox/preloader/preloader.hpp>
namespace keel {
class Context;
}
namespace keel {
struct GbaPlatSpec {
using PtrType = uint32_t;
using size_t = uint32_t;
static constexpr PtrType RomStart = 0x08000000;
[[nodiscard]]
static constexpr std::size_t alignOf(const bool) noexcept {
return 1;
}
[[nodiscard]]
static constexpr std::size_t alignOf(const uint8_t) noexcept {
return 1;
}
[[nodiscard]]
static constexpr std::size_t alignOf(const uint16_t) noexcept {
return 2;
}
[[nodiscard]]
static constexpr std::size_t alignOf(const uint32_t) noexcept {
return 4;
}
[[nodiscard]]
static constexpr std::size_t alignOf(const uint64_t) noexcept {
return 8;
}
[[nodiscard]]
static constexpr std::size_t alignOf(const int8_t) noexcept {
return 1;
}
[[nodiscard]]
static constexpr std::size_t alignOf(const int16_t) noexcept {
return 2;
}
[[nodiscard]]
static constexpr std::size_t alignOf(const int32_t) noexcept {
return 4;
}
[[nodiscard]]
static constexpr std::size_t alignOf(const int64_t) noexcept {
return 8;
}
[[nodiscard]]
static constexpr std::size_t alignOf(const auto*) noexcept {
return 4;
}
[[nodiscard]]
static constexpr auto correctEndianness(auto v) noexcept {
return ox::toLittleEndian(v);
}
};
using GbaPreloader = ox::Preloader<GbaPlatSpec>;
ox::Error appendBinary(ox::Buffer *binBuff, ox::Buffer *fsBuff, GbaPreloader *pl) noexcept;
ox::Error pack(keel::Context *ctx, ox::TypeStore *ts, ox::FileSystem *dest) noexcept;
ox::Error preload(ox::TypeStore *ts, ox::FileSystem *src, GbaPreloader *ph) noexcept;
}

62
src/keel/typeconv.cpp Normal file
View File

@ -0,0 +1,62 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#include <ox/claw/read.hpp>
#include "media.hpp"
#include "typeconv.hpp"
namespace keel {
#ifndef OX_BARE_METAL
[[nodiscard]]
static ox::Result<const BaseConverter*> findConverter(
keel::Context *ctx,
ox::CRStringView srcTypeName, int srcTypeVersion,
ox::CRStringView dstTypeName, int dstTypeVersion) noexcept {
for (auto &c : ctx->converters) {
if (c->matches(srcTypeName, srcTypeVersion, dstTypeName, dstTypeVersion)) {
return c;
}
}
return OxError(1, "Could not find converter");
};
static ox::Result<ox::UniquePtr<Wrap>> convert(
keel::Context *ctx, const ox::Buffer &srcBuffer,
ox::CRStringView srcTypeName, int srcTypeVersion,
ox::CRStringView dstTypeName, int dstTypeVersion) noexcept {
// look for direct converter
auto [c, err] = findConverter(ctx, srcTypeName, srcTypeVersion, dstTypeName, dstTypeVersion);
if (!err) {
return c->convertBuffToPtr(ctx, srcBuffer);
}
// try to chain multiple converters
for (const auto &subConverter : ctx->converters) {
if (!subConverter->dstMatches(dstTypeName, dstTypeVersion)) {
continue;
}
const auto [intermediate, chainErr] =
convert(ctx, srcBuffer, srcTypeName, srcTypeVersion,
subConverter->srcTypeName(), subConverter->srcTypeVersion());
if (!chainErr) {
return subConverter->convertPtrToPtr(ctx, intermediate.get());
}
}
return OxError(1, "Could not convert between types");
}
ox::Result<ox::UniquePtr<Wrap>> convert(
keel::Context *ctx,
const ox::Buffer &srcBuffer,
ox::CRStringView dstTypeName,
int dstTypeVersion) noexcept {
oxRequire(hdr, readAssetHeader(srcBuffer));
return convert(ctx, srcBuffer, hdr.clawHdr.typeName, hdr.clawHdr.typeVersion, dstTypeName, dstTypeVersion);
}
#endif
}

190
src/keel/typeconv.hpp Normal file
View File

@ -0,0 +1,190 @@
/*
* Copyright 2016 - 2023 Gary Talent (gary@drinkingtea.net). All rights reserved.
*/
#pragma once
#include <ox/std/def.hpp>
#include <ox/std/error.hpp>
#include <ox/std/string.hpp>
#include <ox/claw/read.hpp>
#include <ox/claw/write.hpp>
#include "context.hpp"
#include "media.hpp"
namespace keel {
class Wrap {
public:
virtual ~Wrap() = default;
};
template<typename T>
class WrapIndirect: public Wrap {
private:
T *m_obj = nullptr;
public:
template<typename... Args>
constexpr explicit WrapIndirect(Args &&...args): m_obj(ox::forward<Args>(args)...) {
}
[[nodiscard]]
constexpr auto obj() const noexcept {
return &m_obj;
}
[[nodiscard]]
constexpr auto obj() noexcept {
return &m_obj;
}
};
template<typename T>
class WrapInline: public Wrap {
private:
T m_obj;
public:
constexpr WrapInline() = default;
template<typename... Args>
constexpr explicit WrapInline(Args &&...args): m_obj(ox::forward<Args>(args)...) {
}
[[nodiscard]]
constexpr auto obj() noexcept {
return &m_obj;
}
};
template<typename T, typename... Args>
constexpr auto makeWrap(Args &&...args) noexcept {
return ox::make_unique<WrapInline<T>>(ox::forward<Args>(args)...);
}
template<typename T>
constexpr auto wrapCast(auto ptr) noexcept {
return static_cast<WrapInline<T>*>(ptr)->obj();
}
class BaseConverter {
public:
virtual ~BaseConverter() noexcept = default;
[[nodiscard]]
virtual ox::StringView srcTypeName() const noexcept = 0;
[[nodiscard]]
virtual int srcTypeVersion() const noexcept = 0;
[[nodiscard]]
virtual bool srcMatches(ox::CRStringView pSrcTypeName, int pSrcTypeVersion) const noexcept = 0;
[[nodiscard]]
virtual bool dstMatches(ox::CRStringView dstTypeName, int dstTypeVersion) const noexcept = 0;
virtual ox::Result<ox::UniquePtr<Wrap>> convertPtrToPtr(keel::Context *ctx, Wrap *src) const noexcept = 0;
virtual ox::Result<ox::UniquePtr<Wrap>> convertBuffToPtr(keel::Context *ctx, const ox::Buffer &srcBuff) const noexcept = 0;
[[nodiscard]]
inline bool matches(ox::CRStringView srcTypeName, int srcTypeVersion,
ox::CRStringView dstTypeName, int dstTypeVersion) const noexcept {
return srcMatches(srcTypeName, srcTypeVersion)
&& dstMatches(dstTypeName, dstTypeVersion);
}
};
template<typename SrcType, typename DstType>
class Converter: public BaseConverter {
public:
[[nodiscard]]
ox::StringView srcTypeName() const noexcept final {
return ox::requireModelTypeName<SrcType>();
}
[[nodiscard]]
int srcTypeVersion() const noexcept final {
return ox::requireModelTypeVersion<SrcType>();
}
[[nodiscard]]
bool srcMatches(ox::CRStringView pSrcTypeName, int pSrcTypeVersion) const noexcept final {
static const auto SrcTypeName = ox::requireModelTypeName<SrcType>();
static const auto SrcTypeVersion = ox::requireModelTypeVersion<SrcType>();
return pSrcTypeName == SrcTypeName
&& pSrcTypeVersion == SrcTypeVersion;
}
[[nodiscard]]
bool dstMatches(ox::CRStringView dstTypeName, int dstTypeVersion) const noexcept final {
static const auto DstTypeName = ox::StringView(ox::requireModelTypeName<DstType>());
static const auto DstTypeVersion = ox::requireModelTypeVersion<DstType>();
return dstTypeName == DstTypeName
&& dstTypeVersion == DstTypeVersion;
}
ox::Result<ox::UniquePtr<Wrap>> convertPtrToPtr(keel::Context *ctx, Wrap *src) const noexcept final {
auto dst = makeWrap<DstType>();
oxReturnError(convert(ctx, wrapCast<SrcType>(src), wrapCast<DstType>(dst.get())));
return ox::Result<ox::UniquePtr<Wrap>>(std::move(dst));
}
ox::Result<ox::UniquePtr<Wrap>> convertBuffToPtr(keel::Context *ctx, const ox::Buffer &srcBuff) const noexcept final {
oxRequireM(src, readAsset<SrcType>(srcBuff));
auto dst = makeWrap<DstType>();
oxReturnError(convert(ctx, &src, wrapCast<DstType>(dst.get())));
return ox::Result<ox::UniquePtr<Wrap>>(std::move(dst));
}
protected:
virtual ox::Error convert(keel::Context *ctx, SrcType*, DstType*) const noexcept = 0;
};
ox::Result<ox::UniquePtr<Wrap>> convert(keel::Context *ctx, const ox::Buffer &srcBuffer,
ox::CRStringView dstTypeName, int dstTypeVersion) noexcept;
template<typename DstType>
ox::Result<DstType> convert(keel::Context *ctx, const ox::Buffer &srcBuffer) noexcept {
static constexpr auto DstTypeName = ox::requireModelTypeName<DstType>();
static constexpr auto DstTypeVersion = ox::requireModelTypeVersion<DstType>();
oxRequire(out, convert(ctx, srcBuffer, DstTypeName, DstTypeVersion));
return wrapCast<DstType>(out);
}
template<typename DstType>
ox::Error convert(keel::Context *ctx, const ox::Buffer &buff, DstType *outObj) noexcept {
static constexpr auto DstTypeName = ox::requireModelTypeName<DstType>();
static constexpr auto DstTypeVersion = ox::requireModelTypeVersion<DstType>();
oxRequire(outPtr, convert(ctx, buff, DstTypeName, DstTypeVersion));
*outObj = std::move(*wrapCast<DstType>(outPtr.get()));
return OxError(0);
}
template<typename DstType>
ox::Result<ox::Buffer> convertBuffToBuff(keel::Context *ctx, const ox::Buffer &srcBuffer, ox::ClawFormat fmt) noexcept {
static constexpr auto DstTypeName = ox::requireModelTypeName<DstType>();
static constexpr auto DstTypeVersion = ox::requireModelTypeVersion<DstType>();
oxRequire(out, convert(ctx, srcBuffer, DstTypeName, DstTypeVersion));
return ox::writeClaw<DstType>(wrapCast<DstType>(out.get()), fmt);
}
template<typename From, typename To, ox::ClawFormat fmt = ox::ClawFormat::Metal>
auto transformRule(keel::Context *ctx, ox::Buffer *buff) noexcept -> ox::Error {
oxRequire(hdr, readAssetHeader(*buff));
const auto typeId = ox::buildTypeId(
hdr.clawHdr.typeName, hdr.clawHdr.typeVersion, hdr.clawHdr.typeParams);
if (typeId == ox::buildTypeId<From>()) {
oxReturnError(keel::convertBuffToBuff<To>(ctx, *buff, fmt).moveTo(buff));
}
return {};
};
}