index.c

Go to the documentation of this file.

/*
 * File Name: index.c
 */

/*
 * This file is part of ctb.
 *
 * ctb is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 2 of the License, or
 * (at your option) any later version.
 *
 * ctb is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

/**
 * Copyright (C) 2008 iRex Technologies B.V.
 * All rights reserved.
 */

#include "config.h"

#include <sys/time.h>
#include <sys/types.h>
#include <dirent.h>
#include <sys/stat.h>
#include <time.h>
#include <gtk/gtk.h>
#include <unistd.h>
#include <stdlib.h>

#include "log.h"
#include "filetypes.h"
#include "index_db.h"
#include "index.h"
#include "tags.h"
#include "index_ipc.h"
#include "shortcut.h"

const GSList* g_iter;
gchar startdir[256];
int startdirlen = 0;
gchar drz_dir[256];
gchar shortcut_dir[256];
gchar ade_thumbs_dir[256];

typedef struct
{
    const gchar *filename;
    const gchar *filepath;
    gboolean    is_dir;
    gint64      filesize;
    gint64      date_modified;
} fs_entry_t;


typedef unsigned long long u_int64_t;
u_int64_t getCurrentTime()
{
    struct timespec now;

    clock_gettime(CLOCK_MONOTONIC, &now);
    u_int64_t now64 = now.tv_sec;
    now64 *= 1000000;
    now64 += (now.tv_nsec/1000);
    return now64;
}


static const char *HIDDEN_FILENAMES[] =
{
    ERMETADB_GLOBAL_DATABASE_FILE,
    ERMETADB_LOCAL_DATABASE_FILE,
    ERMETADB_LOCAL_DATABASE_FILE "-journal",
    ERMETADB_LOCAL_DATABASE_FILE ".old",
    "autorun.inf",
    "Thumbs.db",
    NULL
};


static gboolean fs_ignore_filename (const gchar *filename)
{
    int len = strlen(filename);
    if (len == 0) return TRUE;

    // check on hidden prefix
    if (filename[0] == '.' || filename[0] == '_') return TRUE;

    // check on hidden filename
    const char** cpp = NULL;
    for (cpp = HIDDEN_FILENAMES ; *cpp; cpp++)
    {
        if ( strcmp(filename, *cpp) == 0 ) return TRUE;
    }

    return FALSE;
}


static db_entry_t* create_db_entry(const fs_entry_t* fs_entry)
{
    db_entry_t* entry = g_malloc(sizeof(db_entry_t));
    memset(entry, 0, sizeof(db_entry_t));
    entry->filename = g_strdup(fs_entry->filename);
    entry->filepath = g_strdup(fs_entry->filepath);
    entry->is_dir = fs_entry->is_dir;
    entry->filesize = fs_entry->filesize;
    entry->last_modified = fs_entry->date_modified;
    entry->state = NEW;
    return entry;
}


static db_entry_t* get_entry(const GSList* db_model, const char* filepath, const char* filename)
{

    const GSList* iter = db_model;
    while (iter) {
        db_entry_t* db_entry = iter->data;
        if ((strcmp(db_entry->filename, filename) == 0) &&
            strcmp(db_entry->filepath, filepath) == 0)
        {
            return db_entry;
        }
        iter = g_slist_next(iter);
    }
    return NULL;
}


static void compare_entry(GSList** db_model, fs_entry_t* fs_entry)
{
    db_entry_t* db_entry = get_entry(*db_model, fs_entry->filepath, fs_entry->filename);
    if (db_entry) {     // entry exists
        if (db_entry->is_dir != fs_entry->is_dir) {
            // add new entry, old entry will be deleted automatically
            db_entry_t* new_entry = create_db_entry(fs_entry);
            // NOTE: entry should be appended! (first delete old one, then add new one)
            *db_model = g_slist_append(*db_model, new_entry);
            return;
        }
        if (!db_entry->is_dir && db_entry->filesize != fs_entry->filesize) {
            db_entry->filesize = fs_entry->filesize;
            db_entry->state = CHANGED;
        }
        if (db_entry->last_modified != fs_entry->date_modified) {
            db_entry->last_modified = fs_entry->date_modified;
            db_entry->state = CHANGED;
        }
        if (db_entry->state == UNKNOWN) db_entry->state = SAME;
    } else {    // unknown entry, add to list
        db_entry_t* new_entry = create_db_entry(fs_entry);
        *db_model = g_slist_prepend(*db_model, new_entry);
        // NOTE: on all entries marked NEW there is memory management on strings!!
    }
}


static gboolean ignore_dir(const char* name)
{
    if (strcmp(ade_thumbs_dir, name) == 0) return TRUE;
    return FALSE;
}


static gboolean ignore_file(const char* filename, const char* ext, gboolean is_dir) 
{
    if (fs_ignore_filename(filename)) return TRUE;

    // if is directory, don't skip
    if (is_dir) return FALSE;

    // ignore files without extension
    if (ext[0] == 0) return TRUE;

    // don't skip shortcuts
    if (is_shortcut_file_extension(ext)) return FALSE;

    return (get_viewer_from_file_extension(ext) == NULL);
}


static gboolean ignore_thumbnail_generation(db_entry_t* db_entry)
{
    // Only handle changed or new files      
    if (db_entry->state != CHANGED && db_entry->state != NEW)
      return TRUE;

    // Ignore directories
    if (db_entry->is_dir)
      return TRUE;

    // Otherwise check extension
    const char *ext = g_extension_pointer(db_entry->filename);
                     
    // Ignore files not configured
    if (get_viewer_from_file_extension(ext) == NULL)
      return TRUE;
      
    // Ignore files where thumbnail generation was not enabled for
    return (!get_generate_thumbnail(ext));
}


static void read_directory(const gchar* name, GSList** db_model)
{
    DIR* dir = opendir(name);
    if (dir == NULL) {
        perror("opendir");
        return;
    }
    struct dirent* current = readdir(dir);
    while (current != 0)
    {
        if (current->d_name[0] == '.') goto next;
        char fullname[1024];
        snprintf(fullname, sizeof(fullname), "%s/%s", name, current->d_name);
        struct stat statbuf;
        stat(fullname, &statbuf);

        gboolean is_dir = S_ISDIR(statbuf.st_mode);
        gboolean is_reg = S_ISREG(statbuf.st_mode);
        if (!is_dir && !is_reg) goto next;

        // try to open and close all local db's for possible version updates
        if (strcmp(current->d_name, ERMETADB_LOCAL_DATABASE_FILE) == 0) {
            LOGPRINTF("trying to open and close %s", fullname);
            erMetadb local_db = ermetadb_local_open(name, TRUE);
            if (local_db == NULL) {
                ERRORPRINTF("cannot open db %s", fullname);
            }
            ermetadb_close(local_db);
            goto next;
        }

        const char *ext = g_extension_pointer(current->d_name);

        if (is_drz_file_extension(ext))
        {
            if (strcmp(name, drz_dir) != 0) {
                // create drz dir if it doesn't exist
                g_mkdir_with_parents(drz_dir, 644);
    
                // move file to drz directory
                char command[256];
                LOGPRINTF("force moving %s to %s/", fullname, drz_dir);
                sprintf(command, "mv -f %s %s/", fullname, drz_dir);
                system(command);
            }
            else
            {
                LOGPRINTF("ignoring %s", current->d_name);
            }
            goto next;
        }

        if (ignore_file(current->d_name, ext, is_dir))
        {
            LOGPRINTF("ignoring %s", current->d_name);
            goto next;
        }

        if (is_dir && ignore_dir(fullname))
        {
            LOGPRINTF("ignoring %s", current->d_name);
            goto next;
        }

        fs_entry_t fs_entry;
        fs_entry.filename = current->d_name;
        fs_entry.filepath = name;
        fs_entry.is_dir = is_dir;
        fs_entry.filesize = is_dir ? 0 : statbuf.st_size;
        fs_entry.date_modified = statbuf.st_mtime;
        compare_entry(db_model, &fs_entry);
        
        if (is_dir)
        { 
            read_directory(fullname, db_model);
        }
next:
        current = readdir(dir);
    }
    closedir(dir);
}

#if (LOGGING_ON)
static const char* state2str(DBState state)
{
    static const char* states[] =
    {
           [UNKNOWN] = "UNKNOWN",
           [SAME]    = "SAME",
           [CHANGED] = "CHANGED",
           [NEW]     = "NEW",
    };
    return states[state];
}
#endif


static gboolean send_store_metadata(db_entry_t* db_entry)
{
     LOGPRINTF("Store metadata for [%s/%s]", db_entry->filepath, db_entry->filename);
     ipc_send_store_metadata(db_entry->filepath, db_entry->filename);
     return TRUE;
}


// display_name and author are ut params, and need to be freed with g_free()
static gboolean check_shortcut(const GSList* db_model,
                               const char* filepath,
                               const char* filename,
                               char** display_name,
                               char** author)
{
    shortcut_t *shortcut = NULL;
    int ret = parse_shortcut_file(filepath, filename, &shortcut);
    if (ret != ER_OK) {
        LOGPRINTF("DENYING: %s/%s - invalid shortcut file", filepath, filename);
        return FALSE;
    }
    g_assert(shortcut);

    if (shortcut->type == SHORTCUT_TO_APPLICATION) {
        // dont check application shortcuts
        // name and author deliberatly empty, will be read from .desktop file on runtime
        *author = g_strdup("");
        *display_name = g_strdup("");
        goto allowed;
    }

    // only allow shortcuts to file or directory (not apps?)
    if (shortcut->type != SHORTCUT_TO_FILE &&
        shortcut->type != SHORTCUT_TO_FOLDER)
    {
        LOGPRINTF("DENYING: %s/%s - not shortcut to file/folder", filepath, filename);
        goto not_allowed;
    }

    // dont allow nested shortcuts
    if (is_shortcut_file(shortcut->details.file.filename)) {
        LOGPRINTF("DENYING: %s/%s - shortcut to shortcut", filepath, filename);
        goto not_allowed;
    }

    // check if destination exists in db
    if (get_entry(db_model, shortcut->details.file.directory,
            shortcut->details.file.filename) == NULL)
    {
        LOGPRINTF("DENYING: %s/%s - target unknown", filepath, filename);
        goto not_allowed;
    }

    // pass ownership to display_name
    *display_name = shortcut->name;
    shortcut->name = NULL;

    // pass target dir relative to startdir (= mountpoint)
    const char* localdir = shortcut->details.file.directory + startdirlen;
    if (localdir[0] == '/') {   // have subdir
        *author = g_strdup_printf("%s/%s", localdir+1, shortcut->details.file.filename);
    } else {        // no subdir
        *author = g_strdup(shortcut->details.file.filename);
    }

allowed:
    shortcut_free(shortcut);
    return TRUE;
not_allowed:
    shortcut_free(shortcut);
    return FALSE;
}


static gboolean handle_changes(const GSList* db_model, const char* dir, gboolean skip_thumbnails)
{
    int removed = 0;
    int same = 0;
    int changed = 0;
    int new = 0;

    db_state_t db_state;
    if (db_open_global(&db_state, dir) != ER_OK) {
        ERRORPRINTF("error opening db");
        return FALSE;
    }

    int ret = db_start_transaction(&db_state);
    if (ret != ER_OK) {
        ERRORPRINTF("error starting transaction");
        goto error;
    }

    const GSList* iter = db_model;
    while (iter) {
        db_entry_t* db_entry = iter->data;
        if (db_entry->state != SAME) {
            LOGPRINTF(" %s  %4lld  %s", state2str(db_entry->state), db_entry->file_id, db_entry->filename);
        }

        gchar* display_name = NULL;
        gchar* author = NULL;

        gboolean is_shortcut = is_shortcut_file(db_entry->filename) && !db_entry->is_dir;
        // special checking for shortcuts
        if (is_shortcut && db_entry->state != UNKNOWN)
        {
            if (!check_shortcut(db_model, db_entry->filepath, db_entry->filename, &display_name, &author)) {
                // delete from fs if it's in DIR_SHORTCUTS
                if (strcmp(db_entry->filepath, shortcut_dir) == 0) {
                    char fullname[256];
                    sprintf(fullname, "%s/%s", db_entry->filepath, db_entry->filename);
                    LOGPRINTF("deleting %s", fullname);
                    int err = unlink(fullname);
                    if (err) {
                        WARNPRINTF("cannot delete %s", fullname);
                    }
                }
                switch (db_entry->state) {
                case UNKNOWN:
                    // was in db, but now deleted, so ok
                    break;
                case SAME:
                case CHANGED:
                    // was in db, delete by setting to unknown
                    db_entry->state = UNKNOWN;
                    break;
                case NEW:
                    // not yet in db, skip it
                    goto next;
                }
            }
        }

        switch (db_entry->state) {
        case UNKNOWN:
            db_delete_entry(&db_state, db_entry);
            removed++;
            break;
        case SAME:
            // do nothing
            same++;
            break;
        case CHANGED:
            db_update_entry(&db_state, db_entry);
            if (is_shortcut) {
                db_update_shortcut_entry(&db_state, db_entry, display_name, author);
            }
            changed++;
            break;
        case NEW:
            {
                const gchar* tag = NULL;
                if (!db_entry->is_dir) tag = get_tag_for_file(db_entry->filename, db_entry->filepath, startdir);
                db_add_entry(&db_state, db_entry, tag, display_name, author);
                new++;
                break;
            }
        }
next:
        g_free(display_name);
        g_free(author);
        iter = g_slist_next(iter);
    }

    db_end_transaction (&db_state);

    if (!skip_thumbnails) {
        // This while loop searches for the first document to store metadata and thumbnails for
        // The rest of the documents is handled in store_metadata_ready
        g_iter = db_model;
        while (g_iter) {
            db_entry_t* db_entry = g_iter->data;

            if (!ignore_thumbnail_generation(db_entry))
            {
                if (send_store_metadata(db_entry))
                {
                    gtk_main();
                    break;
                }
            }
            g_iter = g_slist_next(g_iter);
        }
    }

error:
    db_close(&db_state);
    printf("mdbindex: new %d  removed %d  same %d  changed %d\n", new, removed, same, changed);

    return (new + changed + removed) > 0;
}


/// @brief Handle next documents
///
void store_metadata_ready()
{
    g_iter = g_slist_next(g_iter);

    while (g_iter) {
        db_entry_t* db_entry = g_iter->data;

        if (!ignore_thumbnail_generation(db_entry))
        {
            if (send_store_metadata(db_entry))
            {
                return;
            }
        }
        g_iter = g_slist_next(g_iter);
    }
    gtk_main_quit();
}


gboolean index_full(gchar *dir, gboolean skip_thumbnails)
{
    int len = strlen(dir);
    // strip off trailing slash
    if (len>0 && dir[len-1] == '/') dir[len-1] = '\0';
    strncpy(startdir, dir, sizeof(startdir));
    startdir[sizeof(startdir)-1] = '\0';
    startdirlen = strlen(startdir);

    // check if dir exists and is dir
    struct stat statbuf;
    int res = stat(dir, &statbuf);
    if (res == -1 || !S_ISDIR(statbuf.st_mode)) {
        fprintf(stderr, "mdbindex: directory '%s' doesn't exist or is not a dir\n", dir);
        return FALSE;
    }

    // init static dirs with startdir pre-fix
    sprintf(drz_dir, "%s/%s", startdir, DIR_DRZ);
    sprintf(shortcut_dir, "%s/%s", startdir, DIR_SHORTCUTS);
    sprintf(ade_thumbs_dir, "%s/%s", startdir, DIR_ADE_THUMBS);

    db_state_t db_state;
    if (db_open_global(&db_state, dir) != ER_OK)
    {
        ERRORPRINTF("Failed to open global db");
        return FALSE;
    }
    db_close(&db_state);
    
    filetypes_init(FALSE);

    // read database model
    u_int64_t t1 = getCurrentTime();
    GSList* db_model = db_get_model(dir);
    if (db_model == NULL) WARNPRINTF("DB is empty");
    u_int64_t t2 = getCurrentTime();
    u_int64_t diff = t2 - t1;
    LOGPRINTF("DB DURATION = %lld ms", diff / 1000);

    // read fs and mark changes
    t1 = getCurrentTime();
    read_directory(dir, &db_model);
    t2 = getCurrentTime();
    diff = t2 - t1;
    LOGPRINTF("FS/COMPARE DURATION = %lld ms", diff / 1000);

    // update database with changed entries
    t1 = getCurrentTime();
    gboolean changed = handle_changes(db_model, dir, skip_thumbnails);
    t2 = getCurrentTime();
    diff = t2 - t1;
    LOGPRINTF("DB UPDATE DURATION = %lld ms", diff / 1000);

    // NOTE: not freeing db_model for speed
    return changed;
}