Precount media files to be scanned before they are imported.

.github/workflows/build_ubuntu.yml

@@ -17,4 +17,3 @@ jobs:
        sudo chown root:$USER /etc/fuse.conf
     - run: ./configure
     - run: make
-    - run: make test

src/backup.cc

@@ -1510,6 +1510,9 @@ struct BeakFS : FileSystem
     {
         return false;
     }
+    void allowAccessTimeUpdates()
+    {
+    }
     RC enableWatch()
     {
         return RC::ERR;

src/beak.h

@@ -112,6 +112,7 @@ enum ArgumentType
     ArgStorageOrRule,
     ArgDir,
     ArgFile,
+    ArgFileOrDir,
     ArgFileOrNone,
     ArgORS,  // Origin, Rule or Storage
     ArgNORS, // None, Origin, Rule or Storage
@@ -136,7 +137,7 @@ enum ArgumentType
     X(restore,CommandType::PRIMARY,"Restore from a backup into your file system.",ArgStorage,ArgOrigin) \
     X(shell,CommandType::PRIMARY,"Mount your backup(s) and spawn a shell. Exit the shell to unmount.",ArgStorageOrRule,ArgNone) \
     X(stat,CommandType::PRIMARY,"Show file type statistics for a directory or a backup.",ArgORS,ArgNone) \
-    X(import,CommandType::MEDIA,"Find media files in the source dir then rename and store them normalized into the target dir.",ArgOrigin,ArgStorage) \
+    X(import,CommandType::MEDIA,"Find media files in the source dir then rename and store them normalized into the target dir.",ArgFileOrDir,ArgStorage) \
     X(index,CommandType::MEDIA,"Scan imported media and generate thumbnails and index.html.",ArgOrigin,ArgNone) \
     X(serve,CommandType::MEDIA,"Serve imported media to a web-browser.",ArgOrigin,ArgNone) \
     X(status,CommandType::PRIMARY,"Show the backup status of your configured rules.",ArgRuleOrNone,ArgNone) \
@@ -201,6 +202,7 @@ LIST_OF_OPTIONS
     X(diff_cmd, (1, depth_option) ) \
     X(stat_cmd, (1, depth_option) ) \
     X(fsck_cmd, (1, deepcheck_option) ) \
+    X(import_cmd, (2, include_option, exclude_option) ) \
     X(store_cmd, (14, background_option, contentsplit_option, delta_option, depth_option, splitsize_option, targetsize_option, triggersize_option, triggerglob_option, exclude_option, include_option, padding_option, progress_option, relaxtimechecks_option, tarheader_option, yesorigin_option) ) \
     X(stored_cmd, (14, background_option, contentsplit_option, delta_option, depth_option, splitsize_option, targetsize_option, triggersize_option, triggerglob_option, exclude_option, include_option, padding_option, progress_option, relaxtimechecks_option, tarheader_option, yesorigin_option) ) \
     X(mount_cmd, (3, progress_option,foreground_option, fusedebug_option ) )  \

src/beak_commandline.cc

@@ -1,5 +1,5 @@
 /*
- Copyright (C) 2016-2019 Fredrik Öhrström
+ Copyright (C) 2016-2023 Fredrik Öhrström
 
  This program is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
@@ -79,6 +79,32 @@ Argument BeakImplementation::parseArgument(string arg, ArgumentType expected_typ
         debug(COMMANDLINE, "found point in time (%s) after storage %s\n", point.c_str(), arg.c_str());
     }
 
+    // Check if the argument is a directory or a file.
+    if (expected_type == ArgFileOrDir)
+    {
+        Path *fd = Path::lookup(arg);
+        Path *rp = fd->realpath();
+        if (!rp)
+        {
+            usageError(COMMANDLINE, "Expected file or directory. Got \"%s\" instead.\n", arg.c_str());
+            assert(0);
+        }
+
+        FileStat fs;
+        local_fs_->stat(rp, &fs);
+        if (fs.isDirectory())
+        {
+            argument.dir = rp;
+            argument.type = ArgDir;
+            debug(COMMANDLINE, "found directory arg \"%s\", as expected.\n", fd->c_str());
+            return argument;
+        }
+        argument.file = rp;
+        argument.type = ArgFile;
+        debug(COMMANDLINE, "found file arg \"%s\", as expected.\n", fd->c_str());
+        return argument;
+    }
+
     // Check if the argument is a directory.
     if (expected_type == ArgDir)
     {
@@ -247,6 +273,8 @@ const char *arg_name_(ArgumentType at) {
         return "dir";
     case ArgFile:
         return "file";
+    case ArgFileOrDir:
+        return "file or dir";
     case ArgFileOrNone:
         return "file or none";
     case ArgORS:

src/beak_help.cc

@@ -32,6 +32,7 @@ const char *argName(ArgumentType at) {
     case ArgStorageOrRule: return "<storage>|<rule>";
     case ArgDir: return "<dir>";
     case ArgFile: return "<file>";
+    case ArgFileOrDir: return "<file>|<dir>";
     case ArgFileOrNone: return "[<file>]";
     case ArgORS: return "<origin>|<rule>|<storage>";
     case ArgNORS: return "[<origin>|<rule>|<storage>]";

src/beak_importmedia.cc

@@ -51,6 +51,11 @@ struct ImportMediaData
     {
     }
 
+    void countFile(Path*p, FileStat *st)
+    {
+        db_.countFile(p, st);
+    }
+
     void scanFile(Path *p, FileStat *st, MapFileSystem *map_fs)
     {
         Media *m = db_.addFile(p, st);
@@ -96,28 +101,82 @@ RC BeakImplementation::importMedia(Settings *settings, Monitor *monitor)
 {
     RC rc = RC::OK;
 
-    assert(settings->from.type == ArgOrigin);
+    assert(settings->from.type == ArgDir || settings->from.type == ArgFile);
     assert(settings->to.type == ArgStorage);
 
-    ImportMediaData import_media(this, settings, monitor, local_fs_, sys_);
+    // When importing, do not worry if the access times get updated.
+    local_fs_->allowAccessTimeUpdates();
+
+    std::vector<std::pair<Filter,Match>> filters;
+    for (auto &e : settings->include) {
+        Match m;
+        bool rc = m.use(e);
+        if (!rc) {
+            error(IMPORTMEDIA, "Not a valid glob \"%s\"\n", e.c_str());
+        }
+        filters.push_back(pair<Filter,Match>(Filter(e.c_str(), INCLUDE), m));
+        debug(IMPORTMEDIA, "Includes \"%s\"\n", e.c_str());
+    }
+    for (auto &e : settings->exclude) {
+        Match m;
+        bool rc = m.use(e);
+        if (!rc) {
+            error(IMPORTMEDIA, "Not a valid glob \"%s\"\n", e.c_str());
+        }
+        filters.push_back(pair<Filter,Match>(Filter(e.c_str(), EXCLUDE), m));
+        debug(IMPORTMEDIA, "Excludes \"%s\"\n", e.c_str());
+    }
 
     auto map_fs = newMapFileSystem(local_fs_);
     MapFileSystem *fs = map_fs.get();
 
-    FileStat origin_dir_stat;
-    local_fs_->stat(settings->from.origin, &origin_dir_stat);
-    if (!origin_dir_stat.isDirectory())
-    {
-        usageError(IMPORTMEDIA, "Not a directory: %s\n", settings->from.origin->c_str());
-    }
+    ImportMediaData import_media(this, settings, monitor, local_fs_, sys_);
 
     info(IMPORTMEDIA, "Importing media into %s\n",
          settings->to.storage->storage_location->c_str());
 
-    local_fs_->recurse(settings->from.origin, [&import_media,fs](Path *p, FileStat *st) {
-            import_media.scanFile(p, st, fs);
-            return RecurseOption::RecurseContinue;
-        });
+    if (settings->from.type == ArgDir)
+    {
+        local_fs_->recurse(settings->from.dir, [&import_media,fs,&filters](Path *p, FileStat *st) {
+                int status = 0;
+                for (auto & f : filters) {
+                    bool match  = f.second.match(p->c_str());
+                    int rc = (match)?0:1;
+                    if (f.first.type == INCLUDE) {
+                        status |= rc;
+                    } else {
+                        status |= !rc;
+                    }
+                }
+                if (!status) {
+                    import_media.countFile(p, st);
+                }
+                return RecurseOption::RecurseContinue;
+            });
+
+        local_fs_->recurse(settings->from.dir, [&import_media,fs,&filters](Path *p, FileStat *st) {
+                int status = 0;
+                for (auto & f : filters) {
+                    bool match  = f.second.match(p->c_str());
+                    int rc = (match)?0:1;
+                    if (f.first.type == INCLUDE) {
+                        status |= rc;
+                    } else {
+                        status |= !rc;
+                    }
+                }
+                if (!status) {
+                    import_media.scanFile(p, st, fs);
+                }
+                return RecurseOption::RecurseContinue;
+            });
+    }
+    else
+    {
+        FileStat st;
+        local_fs_->stat(settings->from.file, &st);
+        import_media.scanFile(settings->from.file, &st, fs);
+    }
 
     UI::clearLine();
     string st = import_media.db_.status("ed");

src/filesystem.cc

@@ -51,6 +51,7 @@ struct FileSystemFuseAPIImplementation : FileSystem
     bool createFIFO(Path *path, FileStat *stat);
     bool readLink(Path *file, string *target);
     bool deleteFile(Path *file);
+    void allowAccessTimeUpdates();
     RC mountDaemon(Path *dir, FuseAPI *fuseapi, bool foreground=false, bool debug=false);
     unique_ptr<FuseMount> mount(Path *dir, FuseAPI *fuseapi, bool debug=false);
     RC umount(ptr<FuseMount> fuse_mount);
@@ -210,6 +211,9 @@ bool FileSystemFuseAPIImplementation::deleteFile(Path *path)
     return false;
 }
 
+void FileSystemFuseAPIImplementation::allowAccessTimeUpdates()
+{
+}
 
 size_t basepos(string &s)
 {

src/filesystem.h

@@ -361,6 +361,9 @@ struct FileSystem
 
     virtual bool deleteFile(Path *file) = 0;
 
+    // Its ok if pread will update the access time of the file.
+    virtual void allowAccessTimeUpdates() = 0;
+
     // Enable watching of filesystem changes. Used to warn the user
     // that the filesystem was changed during backup...
     virtual RC enableWatch() = 0;

src/filesystem_helpers.cc

@@ -93,6 +93,10 @@ bool ReadOnlyFileSystem::deleteFile(Path *path)
     return false;
 }
 
+void ReadOnlyFileSystem::allowAccessTimeUpdates()
+{
+}
+
 RC ReadOnlyFileSystem::enableWatch()
 {
     return RC::ERR;

src/filesystem_helpers.h

@@ -45,6 +45,7 @@ struct ReadOnlyFileSystem : FileSystem
     bool createHardLink(Path *path, FileStat *stat, Path *target);
     bool createFIFO(Path *path, FileStat *stat);
     bool deleteFile(Path *file);
+    void allowAccessTimeUpdates();
     RC enableWatch();
     RC addWatch(Path *dir);
     int endWatch();

src/filesystem_posix.cc

@@ -136,6 +136,7 @@ struct FileSystemImplementationPosix : FileSystem
     bool createFIFO(Path *path, FileStat *stat);
     bool readLink(Path *path, string *target);
     bool deleteFile(Path *file);
+    void allowAccessTimeUpdates();
 
     RC enableWatch();
     RC addWatch(Path *dir);
@@ -152,6 +153,7 @@ struct FileSystemImplementationPosix : FileSystem
 
     System *sys_ {};
     Path *temp_dir_;
+    bool allow_access_time_updates_ {};
     //int inotify_fd_ {};
 };
 
@@ -205,16 +207,30 @@ bool FileSystemImplementationPosix::readdir(Path *p, vector<Path*> *vec)
 
 ssize_t FileSystemImplementationPosix::pread(Path *p, char *buf, size_t size, off_t offset)
 {
-    int fd = open(p->c_str(), O_RDONLY | O_NOATIME);
-    if (fd == -1) {
-        // This might be a file not owned by you, if so, open fails if O_NOATIME is enabled.
+    int fd = -1;
+
+    if (allow_access_time_updates_)
+    {
         fd = open(p->c_str(), O_RDONLY);
         if (fd == -1) {
             // Give up permanently.
             return -1;
         }
-        UI::clearLine();
-        info(FILESYSTEM,"You are not the owner of \"%s\" so backing up causes its access time to be updated.\n", p->c_str());
+    }
+    else
+    {
+        // Try to open without updating the access time. This is what you usually want from a backup tool.
+        fd = open(p->c_str(), O_RDONLY | O_NOATIME);
+        if (fd == -1) {
+            // This might be a file not owned by you, if so, open fails if O_NOATIME is enabled.
+            fd = open(p->c_str(), O_RDONLY);
+            if (fd == -1) {
+                // Give up permanently.
+                return -1;
+            }
+            UI::clearLine();
+            info(FILESYSTEM,"You are not the owner of \"%s\" so backing up causes its access time to be updated.\n", p->c_str());
+        }
     }
     ssize_t n = ::pread(fd, buf, size, offset);
     close(fd);
@@ -560,6 +576,11 @@ bool FileSystemImplementationPosix::deleteFile(Path *file)
     return true;
 }
 
+void FileSystemImplementationPosix::allowAccessTimeUpdates()
+{
+    allow_access_time_updates_ = true;
+}
+
 void FileSystemImplementationPosix::initTempDir()
 {
     Path *tmp = Path::lookup(BEAK_SHARED_DIR);

src/media.cc

@@ -670,6 +670,31 @@ bool Media::parseFileName(Path *p)
     return false;
 }
 
+void MediaDatabase::countFile(Path *p, FileStat *st)
+{
+    if (!st->isRegularFile()) return;
+
+    debug(MEDIA, "counting %s\n", p->c_str());
+
+    string ext = p->name()->ext_c_str_();
+
+    if (media_helper_.img_suffixes_.count(ext) != 0)
+    {
+        ext = media_helper_.img_suffixes_[ext];
+        img_suffix_precount_[ext]++;
+    }
+    else if (media_helper_.vid_suffixes_.count(ext) != 0)
+    {
+        ext = media_helper_.vid_suffixes_[ext];
+        vid_suffix_precount_[ext]++;
+    }
+    else if (media_helper_.aud_suffixes_.count(ext) != 0)
+    {
+        ext = media_helper_.aud_suffixes_[ext];
+        aud_suffix_precount_[ext]++;
+    }
+}
+
 bool Media::readFile(Path *p, FileStat *st, FileSystem *fs)
 {
     source_file_ = p;
@@ -790,17 +815,20 @@ string MediaDatabase::status(const char *tense)
     for (auto &p : vid_suffix_count_)
     {
         string s = humanReadable(vid_suffix_size_[p.first]);
-        info += p.first+"("+to_string(p.second)+":"+s+") ";
+        size_t precount = vid_suffix_precount_[p.first];
+        info += p.first+"("+to_string(p.second)+"/"+to_string(precount)+":"+s+") ";
     }
     for (auto &p : img_suffix_count_)
     {
         string s = humanReadable(img_suffix_size_[p.first]);
-        info += p.first+"("+to_string(p.second)+":"+s+") ";
+        size_t precount = img_suffix_precount_[p.first];
+        info += p.first+"("+to_string(p.second)+"/"+to_string(precount)+":"+s+") ";
     }
     for (auto &p : aud_suffix_count_)
     {
         string s = humanReadable(aud_suffix_size_[p.first]);
-        info += p.first+"("+to_string(p.second)+":"+s+") ";
+        size_t precount = aud_suffix_precount_[p.first];
+        info += p.first+"("+to_string(p.second)+"/"+to_string(precount)+":"+s+") ";
     }
     if (unknown_suffix_count_.size() > 0)
     {

src/media.h

@@ -80,6 +80,7 @@ class Media
 class MediaDatabase
 {
 public:
+    void countFile(Path *p, FileStat *st);
     Media *addFile(Path *p, FileStat *st);
     std::string status(const char *tense);
     std::string statusUnknowns();
@@ -99,6 +100,10 @@ MediaDatabase(FileSystem *fs, System *sys) : fs_(fs), sys_(sys) {}
     int num_unknown_files_ {};
     size_t unknown_size_ {};
 
+    std::map<std::string,int> img_suffix_precount_;
+    std::map<std::string,int> vid_suffix_precount_;
+    std::map<std::string,int> aud_suffix_precount_;
+
     std::map<std::string,int> img_suffix_count_;
     std::map<std::string,int> vid_suffix_count_;
     std::map<std::string,int> aud_suffix_count_;