reference.conf

// A default configuration file in HOCON format, almost JSON format
// The file format is described at https://github.com/typesafehub/config.


// Parent directory for data files, downloads, scripts, etc.
baseDir : .


// Directory used for temporary files.
// Override this if you don't have hundreds of GBs free in your system's tmp directory.
tmpDir : ${baseDir}"/.tmp"


// Maximum number of threads that should run simultaneously
// defaults to Runtime.getRuntime().availableProcessors()
maxThreads : -1


// Language sets
// You can specify a custom language set from the command line.
// See EnvBuilder for more information.
languages : {

    // by default use the languages that have local pages
    default : loaded

    // languages that have local pages loaded
    loaded : { type : loaded }

    // languages that have downloaded articles files
    downloaded : { type : downloaded }

    // the largest world economies
    big-economies : {
        type : custom
        langCodes : [
            "en","de","fr","nl","it","pl","es","ru","ja","pt","zh",
            "sv","uk","ca","no","fi","cs","hu","ko","id","tr","ro",
            "sk","da","he","simple"
        ]
    }

    // the english languages
    all-english : {
        type : custom
        langCodes : ["en", "simple"]
    }

    // This can be dynamically populated using the EnvBuilder or command line
    manual : {
        type : custom
        langCodes : []
    }
}


// Filesets downloaded by default
download : {
    matcher : ["articles", "links"]
    path : ${baseDir}"/download"
    listFile : ${download.path}"/list.tsv"
}


// Configuration for the lucene search engine.
lucene : {
    version : "4.3"
    directory : ${baseDir}"/db/lucene"
    options : {
        default : plaintext
        plaintext : {
            type : plaintext

            version : ${lucene.version}
            directory : ${lucene.directory}
            namespaces : ["article"]

            // TokenizerOptions
            caseInsensitive : true
            useStopWords : true
            useStem : true

            // TextFieldElements
            title : 0
            redirects : false
            plaintext : true
        }

        esa : {
            type : esa

            version : ${lucene.version}
            directory : ${lucene.directory}
            namespaces : ["article"]

            // TokenizerOptions
            caseInsensitive : true
            useStopWords : true
            useStem : true

            // TextFieldElements
            title : 1
            redirects : true
            plaintext : true
        }
    }
    searcher : {
        esa : {
            options : esa
        }
    }
}

// multilingual string normalizers
stringnormalizers {
    default : identity

    // do nothing
    identity : {
        type : identity
    }

    // remove punctuation
    simple : {
        type : lucene
        version : ${lucene.version}
        caseInsensitive : false
        useStopWords : false
        useStem : false
    }

    // removes punctuation, folds case
    foldcase : {
        type : lucene
        version : ${lucene.version}
        caseInsensitive : true
        useStopWords : false
        useStem : false
    }

    // fold case, porter stemming
    stemmer : {
        type : lucene
        version : ${lucene.version}
        caseInsensitive : true
        useStopWords : false
        useStem : true
    }
}

// phrase analyzers resolve phrases to articles and vice-versa
phrases {
    // whether or not the phrase analyzers are being loaded.
    // will be overridden while saving corpora to the daos
    loading : false

    // Analyzers
    analyzer : {
        default : fast-cascading
        stanford : {
            phraseDao : stanford
            localPageDao : default
            path : ${download.path}"/stanford-dictionary.bz2"
            url : "http://www-nlp.stanford.edu/pubs/crosswikis-data.tar.bz2/dictionary.bz2"
            type : stanford
            phrasePruner : {
                type : string
                minCount : 3,
                maxRank : 10,
                minFraction : 0.001
            }
            pagePruner : {
                type : simple
                minCount : 3,
                maxRank : 15,
                minFraction : 0.001
            }
        }
        anchortext : {
            phraseDao : anchortext
            localPageDao : default
            localLinkDao : default
            type : anchortext
            phrasePruner : {
                type : string
                minCount : 1,
                maxRank : 10,
                minFraction : 0.001
            }
            pagePruner : {
                type : simple
                minCount : 1,
                maxRank : 15,
                minFraction : 0.001
            }
        }
        lucene : {
            type : lucene
            localPageDao : default
        }
        cascading : {
            type : cascading
            delegates : [ "stanford", "lucene" ]
        }
        fast-cascading : {
            type : cascading
            delegates : [ "anchortext", "lucene" ]
        }
    }

    // Daos store data associated with an algorithm and analyzer
    dao {

        stanford : {
            isNew : ${phrases.loading}
            path : ${baseDir}"/db/phrases/stanford"
            type : objectdb
            normalizer : default
        }
        anchortext : {
            isNew : ${phrases.loading}
            path : ${baseDir}"/db/phrases/anchortext"
            type : objectdb
            normalizer : default
        }
    }
}


// data access objects
dao : {
    dataSource : {
        default : h2
        h2 : {
           driver : org.h2.Driver
           url: "jdbc:h2:"${baseDir}"/db/h2"
           username : sa
           password : ""
        }
        psql : {
           driver : org.h2.Driver
           url: "jdbc:postgresql://localhost/wikAPIdia"
           username : grails
           password : ""
        }
    }
    metaInfo : {
        default : sql
        sql : {
            type : sql
            dataSource : default
        }
        live : {}
    }
    sqlCachePath : ${baseDir}"/db/sql-cache"
    localPage : {
        default : sql
        sql : {
            type : sql
            dataSource : default
        }
        live : {
            type : live
        }

    }
    pageView : {
        default : db
        db : {
            type : db
        }
    }
    localLink : {
        default : matrix
        sql : {
            type : sql
            dataSource : default
        }
        matrix : {
            type : matrix
            delegate : sql
            path : ${baseDir}"/db/matrix/local-link"
        }
        live : {
            type : live
        }
    }
    rawPage : {
        default : sql
        sql : {
            type : sql
            dataSource : default
            localPageDao : sql
        }
        live : {}
    }
    wikidata : {
        default : sql
        sql : {
            type : sql
            dataSource : default
            localPageDao : sql
        }
        live : {}
    }
    universalPage : {
        default : sql
        sql : {
            type : sql
            dataSource : default
        }
        live : {}
    }
    localCategory : {
        sql : {
            type : sql
            dataSource : default
        }
    }
    localArticle : {
        sql : {
            type : sql
            dataSource : default
        }
    }

    localCategoryMember : {
        default : sql
        sql : {
            type : sql
            dataSource: default
        }
        live : {
            type : live
        }
    }

    localArticle : {
        default : sql
        sql : {
            type : sql
            dataSource: default
        }
        live : {
            type : live
        }
    }

    localCategory : {
        default : sql
        sql : {
            type : sql
            dataSource: default
        }
        live : {
            type : live
        }
     }

    universalLink : {
        default : sql
        sql : {
            type : sql
            dataSource : default
            localLinkDao : sql
        }
        skeletal-sql : {
            type : skeletal-sql
            dataSource : default
        }
        live : {}
    }
    redirect : {
        default : sql
        sql : {
            type : sql
            dataSource : default
        }
        live : {
            type : live
        }
    }

}


mapper : {
    default : monolingual
    monolingual : {
        type : monolingual
        algorithmId : 0     // each algorithm must have a unique ID
        localPageDao : sql
    }
    purewikidata : {
        type : purewikidata
        algorithmId : 1
        localPageDao : sql
    }
}


sr : {
    disambig : {
        default : similarity
        topResult : {
            type : topResult
            phraseAnalyzer : default
        }
        topResultConsensus : {
            type : topResultConsensus
            phraseAnalyzers : ["lucene","stanford","anchortext"]
        }
        similarity : {
            type : similarity
            metric : inlinknotrain
            phraseAnalyzer : default

            // how to score candidate senses. Possibilities are:
            //      popularity: just popularity
            //      similarity: just similarity
            //      product: similarity * popularity
            //      sum: similarity + popularity
            criteria : sum
        }
    }

    concepts {
        path : ${baseDir}"/dat/sr/concepts/"
    }

    // The parent configuration for all vector-based SR metrics
    vectorbase {
            type : vector
            pageDao : default
            disambiguator : default

            // Concrete metrics must override the generator
            generator : { type : OVERRIDE_THIS }

            // Default vector similarity is cosine similarity
            similarity : { type : cosine }

            // Method for creating a feature vector for textual phrases
            phrases : {

                // coefficient penalize scores for each type of candidate
                weights : {
                    dab  : 1.0
                    sr   : 0.5
                    text : 0.5
                }

                numCandidates {
                    used  :  1    // number of candidates actually used
                    dab   : 1     // number of disambiguation candidates
                    text  : 0     // number of candidates text heuristic can propose
                    sr    : 0     // number of related candidates sr heuristic can propose
                    perSr : 0     // number of candidates sr heuristic can propose per related candidate
                }

                // lucene analyzer used to find similar text
                lucene : default
            }

            // normalizers
            similaritynormalizer : percentile
            mostsimilarnormalizer : percentile

            // Controls how phrase vectors are created. Values can be:
            //      none: do not create phrase vectors. disambiguate instead.
            //      generator: ask the feature generator to create the phrase vectors
            //      creator: ask the phrase vector create to create the phrase vectors
            //      both: first ask the generator, then the creator
            phraseMode : none
    }

    metric {
        // when training, normalizers are not read from disk
        training : false

        path : ${baseDir}"/dat/sr/"
        local : {
            default : ensemble
            ESA : ${sr.vectorbase} {
                generator : {
                    type : esa
                    luceneSearcher : esa
                    concepts : ${sr.concepts.path}
                }
                similarity : { type : cosine }
                phraseMode : generator
            }
            ESAnotrain : ${sr.vectorbase} {
                generator : {
                    type : esa
                    luceneSearcher : esa
                    concepts : ${sr.concepts.path}
                }
                similarity : { type : cosine }
                similaritynormalizer : identity
                mostsimilarnormalizer : identity
                phraseMode : generator
            }
            outlink : ${sr.vectorbase} {
                generator : {
                    type : links
                    outLinks : true
                }
                similarity : {
                    type : google
                }
            }
            inlink : ${sr.vectorbase} {
                generator : {
                    type : links
                    outLinks : false
                }
                similarity : {
                    type : google
                }
            }
            inlinknotrain : ${sr.vectorbase} {
                generator : {
                    type : links
                    outLinks : false
                }
                similarity : {
                    type : google
                }
                similaritynormalizer : identity
                mostsimilarnormalizer : identity
            }
            fast-ensemble : {
                type : ensemble
                metrics : ["milnewitten","milnewittenout"]
                similaritynormalizer : identity
                mostsimilarnormalizer : identity
                ensemble : linear
                disambiguator : default
                pageDao : default
            }
            ensemble : {
                type : ensemble
                metrics : ["ESA","inlink","outlink","categorygraphsimilarity"]
                similaritynormalizer : percentile
                mostsimilarnormalizer : percentile
                ensemble : linear
                resolvephrases : false
                disambiguator : default
                pageDao : default
            }
            super-ensemble : {
                type : ensemble
                metrics : ["ESA","inlink","outlink","categorygraphsimilarity","mostsimilarcosine"]
                similaritynormalizer : percentile
                mostsimilarnormalizer : percentile
                ensemble : linear
                resolvephrases : false
                disambiguator : default
                pageDao : default
            }
            mostsimilarcosine : ${sr.vectorbase} {
                generator : {
                    type : mostsimilarconcepts
                    basemetric : ensemble
                    concepts : ${sr.concepts.path}
                }
            }
            categorysimilarity :{
                type : categorysimilarity
                disambiguator : default
                pageDao : default
                categoryMemberDao : default
                similaritynormalizer : percentile
                mostsimilarnormalizer : percentile
            }
            categorygraphsimilarity :{
                type : categorygraphsimilarity
                disambiguator : default
                pageDao : default
                categoryMemberDao : default
                similaritynormalizer : percentile
                mostsimilarnormalizer : percentile
            }
        }
        universal : {
            default : UniversalMilneWitten
            UniversalMilneWitten : {
                type: UniversalMilneWitten
                disambiguator : default
                linkDao : default
                pageDao : default
                outLinks : false
                similaritynormalizer : loess
                mostsimilarnormalizer : rank
            }
        }
    }

    ensemble {
        default : linear
        even : {
            type : even
        }
        linear : {
            type : linear
        }
    }

    normalizer {
        defaultmaxresults : 100
        identity : {
            type : identity
        }
        logLoess : {
            type : loess
            log : true
        }
        loess : {
            type : loess
        }
        log : {
            type : log
        }
        percentile : {
            type : percentile
        }
        range : {
            type : range
            min : 0.0
            max : 1.0
            truncate : true
        }
        rank : {
            type : rank
        }
    }

    explanationformatter {
        explanationformatter {
            localpagedao : sql
        }
    }

    dataset : {
        dao : {
            resource : {
                type : resource
                disambig : topResult
                resolvePhrases : true
            }
        }
        defaultsets : ["wordsim353.txt","MC.txt"]
        groups : {
                // large, commonly used datasets
                major-en : ["wordsim353.txt", "MTURK-771.csv", "atlasify240.txt", "radinsky.txt"]
        }
        // pairs under this threshold won't be used for most similar training.
        mostSimilarThreshold : 0.7
        records : ${baseDir}"/dat/records/"
    }

}

loader {
    // Stages of the loading pipeline, used by PipelineLoader
    stages : [
            {
                name : fetchlinks,
                class : org.wikapidia.download.RequestedLinkGetter
                onByDefault : true
                extraArgs : [],
            },
            {
                name : download,
                class : org.wikapidia.download.DumpFileDownloader,
                onByDefault : true
                extraArgs : [],
            },
            {
                name : dumploader,
                class : org.wikapidia.dao.load.DumpLoader,
                onByDefault : true
                extraArgs : ["-d"],
            },
            {
                name : redirects,
                class : org.wikapidia.dao.load.RedirectLoader,
                onByDefault : true
                extraArgs : ["-d"],
            },
            {
                name : wikitext,
                class : org.wikapidia.dao.load.WikiTextLoader,
                onByDefault : true
                extraArgs : ["-d"],
            },
            {
                name : sqllinks,
                class : org.wikapidia.dao.load.SqlLinksLoader,
                onByDefault : true
                extraArgs : [],
            },
            {
                name : lucene,
                class : org.wikapidia.dao.load.LuceneLoader,
                onByDefault : true
                extraArgs : [],
            },
            {
                name : concept,
                class : org.wikapidia.dao.load.ConceptLoader,
                onByDefault : true
                extraArgs : ["-d"],
            },
            {
                name : universal,
                class : org.wikapidia.dao.load.UniversalLinkLoader,
                onByDefault : true
                extraArgs : ["-d"],
            },
            {
                name : phrase,
                class : org.wikapidia.dao.load.PhraseLoader,
                onByDefault : true
                extraArgs : ["-p", "anchortext"],
            },
            {
                name : wikidata,
                class : org.wikapidia.wikidata.WikidataDumpLoader,
                onByDefault : false
                extraArgs : ["-d"],
            }
    ]
}


// backup for integration tests
integration {
    dir : ${baseDir}"/backup"
}