{

bool operator()(const LightFileFingerprint& lhs, const LightFileFingerprint& rhs) const

{

return LightFileFingerprintCmp{}(&lhs, &rhs);

}

{

handle fsid;

LocalPath path;

{

using FingerprintSet = std::set<LightFileFingerprint, LightFileFingerprintComparator>;

// Adds a new fingerprint

template<typename T, typename = typename std::enable_if<std::is_same<LightFileFingerprint, typename std::decay<T>::type>::value>::type>

const LightFileFingerprint* add(T&& ffp)

{

const auto insertPair = mFingerprints.insert(std::forward<T>(ffp));

return &*insertPair.first;

}

// Returns the set of all fingerprints

const FingerprintSet& all() const

{

return mFingerprints;

}

FingerprintSet mFingerprints;

LocalPath& localdebris)

{

auto fa = fsaccess.newfileaccess(false);

if (!fa->fopen(localpath, true, false))

{

LOG_err << "Unable to open path: " << localpath.toPath(fsaccess);

return {};

}

if (fa->mIsSymLink)

{

LOG_debug << "Ignoring symlink: " << localpath.toPath(fsaccess);

return {};

}

assert(fa->type == FOLDERNODE);

auto da = std::unique_ptr<DirAccess>{fsaccess.newdiraccess()};

if (!da->dopen(&localpath, fa.get(), false))

{

LOG_err << "Unable to open directory: " << localpath.toPath(fsaccess);

return {};

}

set<LocalPath> paths; // has to be a std::set to enforce same sorting as `children` of `LocalNode`

LocalPath localname;

while (da->dnext(localpath, localname, false))

{

ScopedLengthRestore restoreLength(localpath);

localpath.appendWithSeparator(localname, false, fsaccess.localseparator);

// check if this record is to be ignored

if (app.sync_syncable(&sync, localname.toName(fsaccess).c_str(), localpath))

{

// skip the sync's debris folder

if (!localdebris.isContainingPathOf(localpath, fsaccess))

{

paths.insert(localpath);

}

}

}

return paths;

{

hashCombine(ffp.size, other.size);

hashCombine(ffp.mtime, other.mtime);

{

bool success = false;

for (const auto& nodePair : nodeMap)

{

const LocalNode& l = *nodePair.second;

if (l.type == FILENODE)

{

LightFileFingerprint lFfp;

lFfp.genfingerprint(l.size, l.mtime);

hashCombineFingerprint(ffp, lFfp);

success = true;

}

}

return success;

{

bool success = false;

for (auto& path : paths)

{

auto fa = fsaccess.newfileaccess(false);

auto pathArg = path; // todo: sort out const

if (!fa->fopen(pathArg, true, false))

{

LOG_err << "Unable to open path: " << path.toPath(fsaccess);

success = false;

break;

}

if (fa->mIsSymLink)

{

LOG_debug << "Ignoring symlink: " << path.toPath(fsaccess);

continue;

}

if (fa->type == FILENODE)

{

LightFileFingerprint faFfp;

faFfp.genfingerprint(fa->size, fa->mtime);

hashCombineFingerprint(ffp, faFfp);

success = true;

}

}

return success;

{

if (l.type == FILENODE)

{

ffp.genfingerprint(l.size, l.mtime);

return true;

}

else if (l.type == FOLDERNODE)

{

return combinedFingerprint(ffp, l.children);

}

else

{

assert(false && "Invalid node type");

return false;

}

FileAccess& fa, LocalPath& path, const set<LocalPath>& paths)

{

if (fa.type == FILENODE)

{

assert(paths.empty());

ffp.genfingerprint(fa.size, fa.mtime);

return true;

}

else if (fa.type == FOLDERNODE)

{

return combinedFingerprint(ffp, fsaccess, paths);

}

else

{

assert(false && "Invalid node type");

return false;

}

LocalNode& l, handlelocalnode_map& fsidnodes)

{

// invalidate fsid of `l`

l.fsid = mega::UNDEF;

if (l.fsid_it != fsidnodes.end())

{

fsidnodes.erase(l.fsid_it);

l.fsid_it = fsidnodes.end();

}

// collect fingerprint

LightFileFingerprint ffp;

if (computeFingerprint(ffp, l))

{

const auto ffpPtr = fingerprints.add(std::move(ffp));

localnodes.insert(std::make_pair(ffpPtr, &l));

}

if (l.type == FILENODE)

{

return;

}

for (auto& childPair : l.children)

{

collectAllLocalNodes(fingerprints, localnodes, *childPair.second, fsidnodes);

}

Sync& sync, MegaApp& app, FileSystemAccess& fsaccess, LocalPath& localpath,

LocalPath& localdebris)

{

auto insertFingerprint = [&files, &fingerprints](FileSystemAccess& fsaccess, FileAccess& fa,

LocalPath& path, const set<LocalPath>& paths)

{

LightFileFingerprint ffp;

if (computeFingerprint(ffp, fsaccess, fa, path, paths))

{

const auto ffpPtr = fingerprints.add(std::move(ffp));

files.insert(std::make_pair(ffpPtr, FsFile{fa.fsid, path}));

}

};

auto fa = fsaccess.newfileaccess(false);

if (!fa->fopen(localpath, true, false))

{

LOG_err << "Unable to open path: " << localpath.toPath(fsaccess);

success = false;

return;

}

if (fa->mIsSymLink)

{

LOG_debug << "Ignoring symlink: " << localpath.toPath(fsaccess);

return;

}

if (!fa->fsidvalid)

{

LOG_err << "Invalid fs id for: " << localpath.toPath(fsaccess);

success = false;

return;

}

if (fa->type == FILENODE)

{

insertFingerprint(fsaccess, *fa, localpath, {});

}

else if (fa->type == FOLDERNODE)

{

const auto paths = collectAllPathsInFolder(sync, app, fsaccess, localpath, localdebris);

insertFingerprint(fsaccess, *fa, localpath, paths);

fa.reset();

for (const auto& path : paths)

{

LocalPath tmpPath = path;

collectAllFiles(success, fingerprints, files, sync, app, fsaccess, tmpPath, localdebris);

}

}

else

{

assert(false && "Invalid file type");

success = false;

return;

}

FingerprintFileMap& files, handlelocalnode_map& fsidnodes, FileSystemAccess& fsaccess)

{

LocalPath nodePath;

string accumulated;

size_t assignmentCount = 0;

for (const auto& fp : fingerprints.all())

{

const auto nodeRange = localnodes.equal_range(&fp);

const auto nodeCount = std::distance(nodeRange.first, nodeRange.second);

if (nodeCount <= 0)

{

continue;

}

const auto fileRange = files.equal_range(&fp);

const auto fileCount = std::distance(fileRange.first, fileRange.second);

if (fileCount <= 0)

{

// without files we cannot assign fs IDs to these localnodes, so no need to keep them

localnodes.erase(nodeRange.first, nodeRange.second);

continue;

}

struct Element

{

int score;

handle fsid;

LocalNode* l;

};

std::vector<Element> elements;

elements.reserve(nodeCount * fileCount);

for (auto nodeIt = nodeRange.first; nodeIt != nodeRange.second; ++nodeIt)

{

auto l = nodeIt->second;

if (l != l->sync->localroot.get()) // never assign fs ID to the root localnode

{

nodePath = l->getLocalPath(false);

for (auto fileIt = fileRange.first; fileIt != fileRange.second; ++fileIt)

{

auto& filePath = fileIt->second.path;

const auto score = computeReversePathMatchScore(accumulated, nodePath, filePath, fsaccess);

if (score > 0) // leaf name must match

{

elements.push_back({score, fileIt->second.fsid, l});

}

}

}

}

// Sort in descending order by score. Elements with highest score come first

std::sort(elements.begin(), elements.end(), [](const Element& e1, const Element& e2)

{

return e1.score > e2.score;

});

std::unordered_set<handle> usedFsIds;

for (const auto& e : elements)

{

if (e.l->fsid == mega::UNDEF // node not assigned

&& usedFsIds.find(e.fsid) == usedFsIds.end()) // fsid not used

{

e.l->setfsid(e.fsid, fsidnodes);

usedFsIds.insert(e.fsid);

++assignmentCount;

}

}

// the fingerprint that these files and localnodes correspond to has now finished processing

files.erase(fileRange.first, fileRange.second);

localnodes.erase(nodeRange.first, nodeRange.second);

}

return assignmentCount;

{

const string& path1 = *path1Arg.editStringDirect();

const string& path2 = *path2Arg.editStringDirect();

if (path1.empty() || path2.empty())

{

return 0;

}

accumulated.clear();

const auto path1End = path1.size() - 1;

const auto path2End = path2.size() - 1;

size_t index = 0;

size_t separatorBias = 0;

while (index <= path1End && index <= path2End)

{

const auto value1 = path1[path1End - index];

const auto value2 = path2[path2End - index];

if (value1 != value2)

{

break;

}

accumulated.push_back(value1);

++index;

if (accumulated.size() >= fsaccess.localseparator.size())

{

const auto diffSize = accumulated.size() - fsaccess.localseparator.size();

if (std::equal(accumulated.begin() + diffSize, accumulated.end(), fsaccess.localseparator.begin()))

{

separatorBias += fsaccess.localseparator.size();

accumulated.clear();

}

}

}

if (index > path1End && index > path2End) // we got to the beginning of both paths (full score)

{

return static_cast<int>(index - separatorBias);

}

else // the paths only partly match

{

return static_cast<int>(index - separatorBias - accumulated.size());

}

LocalPath& localdebris)

{

auto& rootpath = sync.localroot->localname;

LOG_info << "Assigning fs IDs at rootpath: " << rootpath.toPath(fsaccess);

auto fa = fsaccess.newfileaccess(false);

if (!fa->fopen(rootpath, true, false))

{

LOG_err << "Unable to open rootpath";

return false;

}

if (fa->type != FOLDERNODE)

{

LOG_err << "rootpath not a folder";

assert(false);

return false;

}

if (fa->mIsSymLink)

{

LOG_err << "rootpath is a symlink";

assert(false);

return false;

}

fa.reset();

bool success = true;

FingerprintCache fingerprints;

FingerprintLocalNodeMap localnodes;

collectAllLocalNodes(fingerprints, localnodes, *sync.localroot, fsidnodes);

LOG_info << "Number of localnodes: " << localnodes.size();

if (localnodes.empty())

{

return success;

}

FingerprintFileMap files;

collectAllFiles(success, fingerprints, files, sync, app, fsaccess, rootpath, localdebris);

LOG_info << "Number of files: " << files.size();

LOG_info << "Number of fingerprints: " << fingerprints.all().size();

const auto assignmentCount = assignFilesystemIdsImpl(fingerprints, localnodes, files, fsidnodes, fsaccess);

LOG_info << "Number of fsid assignments: " << assignmentCount;

return success;

{

std::string dbname = "syncconfigs_" + id;

mTable.reset(dbaccess.open(rng, &fsaccess, &dbname, false, false));

if (!mTable)

{

LOG_err << "Unable to open DB table: " << dbname;

assert(false);

return;

}

mTable->rewind();

uint32_t tableId;

std::string data;

while (mTable->next(&tableId, &data))

{

auto syncConfig = SyncConfig::unserialize(data);

if (!syncConfig)

{

LOG_err << "Unable to unserialize sync config at id: " << tableId;

assert(false);

continue;

}

syncConfig->dbid = tableId;

mSyncConfigs.insert(std::make_pair(syncConfig->getLocalPath(), *syncConfig));

if (tableId > mTable->nextid)

{

mTable->nextid = tableId;

}

}

++mTable->nextid;

{

auto insertOrUpdate = [this](const uint32_t id, const SyncConfig& syncConfig)

{

std::string data;

const_cast<SyncConfig&>(syncConfig).serialize(&data);

DBTableTransactionCommitter committer{mTable.get()};

if (!mTable->put(id, &data)) // put either inserts or updates

{

LOG_err << "Incomplete database put at id: " << mTable->nextid;

assert(false);

mTable->abort();

return false;

}

return true;

};

map<string, SyncConfig>::iterator syncConfigIt = mSyncConfigs.find(syncConfig.getLocalPath());

if (syncConfigIt == mSyncConfigs.end()) // syncConfig is new

{

if (mTable)

{

if (!insertOrUpdate(mTable->nextid, syncConfig))

{

return;

}

}

auto insertPair = mSyncConfigs.insert(std::make_pair(syncConfig.getLocalPath(), syncConfig));

if (mTable)

{

insertPair.first->second.dbid = mTable->nextid;

++mTable->nextid;

}

}

else // syncConfig exists already

{

const uint32_t tableId = syncConfigIt->second.dbid;

if (mTable)

{

if (!insertOrUpdate(tableId, syncConfig))

{

return;

}

}

syncConfigIt->second = syncConfig;

syncConfigIt->second.dbid = tableId;

}

{

auto syncConfigPair = mSyncConfigs.find(localPath);

if (syncConfigPair != mSyncConfigs.end())

{

if (mTable)

{

DBTableTransactionCommitter committer{mTable.get()};

if (!mTable->del(syncConfigPair->second.dbid))

{

LOG_err << "Incomplete database del at id: " << syncConfigPair->second.dbid;

assert(false);

mTable->abort();

return;

}

}

mSyncConfigs.erase(syncConfigPair);

}

{

if (mTable)

{

mTable->truncate();

mTable->nextid = 0;

}

mSyncConfigs.clear();

{

std::vector<SyncConfig> syncConfigs;

for (const auto& syncConfigPair : mSyncConfigs)

{

syncConfigs.push_back(syncConfigPair.second);

}

return syncConfigs;

string* clocaldebris, Node* remotenode, bool cinshare, int ctag, void *cappdata)

: localroot(new LocalNode)

{

isnetwork = false;

client = cclient;

tag = ctag;

inshare = cinshare;

appData = cappdata;

errorcode = API_OK;

tmpfa = NULL;

initializing = true;

updatedfilesize = ~0;

updatedfilets = 0;

updatedfileinitialts = 0;

localbytes = 0;

localnodes[FILENODE] = 0;

localnodes[FOLDERNODE] = 0;

state = SYNC_INITIALSCAN;

statecachetable = NULL;

fullscan = true;

scanseqno = 0;

mLocalPath = config.getLocalPath();

LocalPath crootpath = LocalPath::fromPath(mLocalPath, *client->fsaccess);

if (cdebris)

{

debris = cdebris;

localdebris = LocalPath::fromPath(debris, *client->fsaccess);

dirnotify.reset(client->fsaccess->newdirnotify(crootpath, localdebris, client->waiter));

localdebris.prependWithSeparator(crootpath, client->fsaccess->localseparator);

}

else

{

localdebris = LocalPath::fromLocalname(*clocaldebris);

// FIXME: pass last segment of localdebris

dirnotify.reset(client->fsaccess->newdirnotify(crootpath, localdebris, client->waiter));

}

dirnotify->sync = this;

// set specified fsfp or get from fs if none

const auto cfsfp = config.getLocalFingerprint();

if (cfsfp)

{

fsfp = cfsfp;

}

else

{

fsfp = dirnotify->fsfingerprint();

config.setLocalFingerprint(fsfp);

}

fsstableids = dirnotify->fsstableids();

LOG_info << "Filesystem IDs are stable: " << fsstableids;

mFilesystemType = client->fsaccess->getFilesystemType(crootpath);

localroot->init(this, FOLDERNODE, NULL, crootpath, nullptr); // the root node must have the absolute path. We don't store shortname, to avoid accidentally using relative paths.

localroot->setnode(remotenode);

#ifdef __APPLE__

if (macOSmajorVersion() >= 19) //macOS catalina+

{

LOG_debug << "macOS 10.15+ filesystem detected. Checking fseventspath.";

string supercrootpath = "/System/Volumes/Data" + *crootpath.editStringDirect();

int fd = open(supercrootpath.c_str(), O_RDONLY);

if (fd == -1)

{

LOG_debug << "Unable to open path using fseventspath.";

mFsEventsPath = *crootpath.editStringDirect();

}

else

{

char buf[MAXPATHLEN];

if (fcntl(fd, F_GETPATH, buf) < 0)

{

LOG_debug << "Using standard paths to detect filesystem notifications.";

mFsEventsPath = *crootpath.editStringDirect();

}

else

{

LOG_debug << "Using fsevents paths to detect filesystem notifications.";

mFsEventsPath = supercrootpath;

}

close(fd);

}

}

#endif

sync_it = client->syncs.insert(client->syncs.end(), this);

if (client->dbaccess)

{

// open state cache table

handle tableid[3];

string dbname;

auto fas = client->fsaccess->newfileaccess(false);

if (fas->fopen(crootpath, true, false))

{

tableid[0] = fas->fsid;

tableid[1] = remotenode->nodehandle;

tableid[2] = client->me;

dbname.resize(sizeof tableid * 4 / 3 + 3);

dbname.resize(Base64::btoa((byte*)tableid, sizeof tableid, (char*)dbname.c_str()));

statecachetable = client->dbaccess->open(client->rng, client->fsaccess, &dbname, false, false);

readstatecache();

}

}

if (client->syncConfigs)

{

client->syncConfigs->insert(config);

}

{

auto range = tmap->equal_range(parent_dbid);

for (auto it = range.first; it != range.second; it++)

{

ScopedLengthRestore restoreLen(localpath);

localpath.appendWithSeparator(it->second->localname, true, client->fsaccess->localseparator);

LocalNode* l = it->second;

Node* node = l->node;

handle fsid = l->fsid;

m_off_t size = l->size;

// clear localname to force newnode = true in setnameparent

l->localname.clear();

// if we already have the shortname from database, use that, otherwise (db is from old code) look it up

std::unique_ptr<LocalPath> shortname;

if (l->slocalname_in_db)

{

// null if there is no shortname, or the shortname matches the localname.

shortname.reset(l->slocalname.release());

}

else

{

shortname = client->fsaccess->fsShortname(localpath);

}

l->init(this, l->type, p, localpath, std::move(shortname));

#ifdef DEBUG

auto fa = client->fsaccess->newfileaccess(false);

if (fa->fopen(localpath)) // exists, is file

{

auto sn = client->fsaccess->fsShortname(localpath);

assert(!l->localname.empty() &&

(!l->slocalname && (!sn || l->localname == *sn) ||

(l->slocalname && sn && !l->slocalname->empty() && *l->slocalname != l->localname && *l->slocalname == *sn)));

}

#endif

l->parent_dbid = parent_dbid;

l->size = size;

l->setfsid(fsid, client->fsidnode);

l->setnode(node);

if (!l->slocalname_in_db)

{

statecacheadd(l);

if (insertq.size() > 50000)

{

cachenodes(); // periodically output updated nodes with shortname updates, so people who restart megasync still make progress towards a fast startup

}

}

if (maxdepth)

{

addstatecachechildren(l->dbid, tmap, localpath, l, maxdepth - 1);

}

}

{

if (statecachetable && state == SYNC_INITIALSCAN)

{

string cachedata;

idlocalnode_map tmap;

uint32_t cid;

LocalNode* l;

statecachetable->rewind();

// bulk-load cached nodes into tmap

while (statecachetable->next(&cid, &cachedata, &client->key))

{

if ((l = LocalNode::unserialize(this, &cachedata)))

{

l->dbid = cid;

tmap.insert(pair<int32_t,LocalNode*>(l->parent_dbid,l));

}

}

// recursively build LocalNode tree, set scanseqnos to sync's current scanseqno

addstatecachechildren(0, &tmap, localroot->localname, localroot.get(), 100);

cachenodes();

// trigger a single-pass full scan to identify deleted nodes

fullscan = true;

scanseqno++;

return true;

}

return false;

{

if (client->syncConfigs)

{

const auto config = client->syncConfigs->get(mLocalPath);

assert(config);

auto newConfig = *config;

newConfig.setResumable(isResumable);

client->syncConfigs->insert(newConfig);

}

{

if (state == SYNC_CANCELED)

{

return;

}

insertq.erase(l);

if (l->dbid)

{

deleteq.insert(l->dbid);

}

{

if (state == SYNC_CANCELED)

{

return;

}

if (l->dbid)

{

deleteq.erase(l->dbid);

}

insertq.insert(l);

{

if (statecachetable && (state == SYNC_ACTIVE || (state == SYNC_INITIALSCAN && insertq.size() > 100)) && (deleteq.size() || insertq.size()))

{

LOG_debug << "Saving LocalNode database with " << insertq.size() << " additions and " << deleteq.size() << " deletions";

statecachetable->begin();

// deletions

for (set<uint32_t>::iterator it = deleteq.begin(); it != deleteq.end(); it++)

{

statecachetable->del(*it);

}

deleteq.clear();

// additions - we iterate until completion or until we get stuck

bool added;

do {

added = false;

for (set<LocalNode*>::iterator it = insertq.begin(); it != insertq.end(); )

{

if ((*it)->parent->dbid || (*it)->parent == localroot.get())

{

statecachetable->put(MegaClient::CACHEDLOCALNODE, *it, &client->key);

insertq.erase(it++);

added = true;

}

else it++;

}

} while (added);

statecachetable->commit();

if (insertq.size())

{

LOG_err << "LocalNode caching did not complete";

}

}

{

if (newstate != state)

{

client->app->syncupdate_state(this, newstate);

if (newstate == SYNC_FAILED && statecachetable)

{

statecachetable->remove();