Elasticsearch integrations for git repositories.
Now indexing text-like documents mo nore 1 mb size
Add this line to your application's Gemfile:
gem 'elasticsearch-git', '~> 0.0.5'
# or
gem 'elasticsearch-git', github: 'zzet/elasticsearch-git', ref: 'last_ref'
And then execute:
$ bundle
Or install it yourself as:
$ gem install elasticsearch-git
class Repository
include Elasticsearch::Git::Repository
def repository_id
project.id
end
repository_for_indexing '/path/to/your/repo'
end
Repository.__elasticsearch__.create_index! force: true
repo = Repository.new
repo.index_commits
repo.index_blobs
repo.index_commits(from_rev: "1802bafa70d3b1678cfa46a482fd396dd8a4bd40", to_rev: "8d4175e9f4a36065b52fa752c1fd3594c82c0f28")
repo.index_blobs(from_rev: "1802bafa70d3b1678cfa46a482fd396dd8a4bd40", to_rev: "8d4175e9f4a36065b52fa752c1fd3594c82c0f28")
Repository.search("query", type: 'blob')
Repository.search("query", type: 'commit')
# Search in all types
Repository.search("query")
# app/elastic/repositories_search.rb
module RepositoriesSearch
extend ActiveSupport::Concern
included do
include Elasticsearch::Git::Repository
self.__elasticsearch__.client = Elasticsearch::Client.new host: Gitlab.config.elasticsearch.host, port: Gitlab.config.elasticsearch.port
def repository_id
project.id
end
end
module ClassMethods
def import
Repository.__elasticsearch__.create_index! force: true
Project.find_each do |project|
if project.repository.exists? && !project.repository.empty?
project.repository.index_commits
project.repository.index_blobs
end
end
end
end
end
# app/models/repository.rb
class Repository
include RepositoriesSearch
#...
def project
@project ||= Project.find_with_namespace(@path_with_namespace)
end
#...
end
Repository.import # for indexing all repositories
Repository.__elasticsearch__.create_index! force: true
Project.last.repository.index_commits
Project.last.repository.index_blobs
Repository.search("some_query")
# => {blobs: [{}, {}, {}], commits: [{}, {}, {}]}
Repository.search("some_query", type: :blob)
# => {blobs: [{}, {}, {}], commits: []}
Repository.search("some_query", type: :commit)
# => {blobs: [], commits: [{}, {}, {}]}
Repository.search("some_query", type: :commit, page: 2, per: 50)
# => ...
Repository.search("some_query", options: { repository_id: Project.last.id })
# => {blobs: [{}, {}, {}], commits: [{}, {}, {}]}
Repository.search("some_query", options: { repository_id: current_user.authorized_projects.ids })
# => {blobs: [{}, {}, {}], commits: [{}, {}, {}]}
Project.last.repository.search("Copyright")[:blobs].first
=> #<Elasticsearch::Model::Response::Result:0xbb84b3fc
@result=
{"_index"=>"repository-index-development",
"_type"=>"repository",
"_id"=>"4328_LICENSE.txt",
"_score"=>0.034848917,
"_source"=>
{"blob"=>
{"type"=>"blob",
"oid"=>"f99909cd4ecb6f2ad08f8e55aac3a9fcd86a2bd2",
"rid"=>4328,
"content"=>
"Copyright (c) 2014 Andrey Kumanyaev\n\nMIT
License\n\nPermission is hereby granted, free of charge, to any person
obtaining\na copy of this software and associated documentation files
(the\n\"Software\"), to deal in the Software without restriction,
including\nwithout limitation the rights to use, copy, modify, merge,
publish,\ndistribute, sublicense, and/or sell copies of the Software,
and to\npermit persons to whom the Software is furnished to do so,
subject to\nthe following conditions:\n\nThe above copyright notice and
this permission notice shall be\nincluded in all copies or substantial
portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT
WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE
AND\nNONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
BE\nLIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.\n",
"commit_sha"=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"}}}>
After integration this gem into Gitlab
Repository.search("too")[:commits].first
=> #<Elasticsearch::Model::Response::Result:0xbb50dfdc
@result=
{"_index"=>"repository-index-development",
"_type"=>"repository",
"_id"=>"4328_1802bafa70d3b1678cfa46a482fd396dd8a4bd40",
"_score"=>0.15873253,
"_source"=>
{"commit"=>
{"type"=>"commit",
"rid"=>4328,
"sha"=>"1802bafa70d3b1678cfa46a482fd396dd8a4bd40",
"author"=>
{"name"=>"Andrey Kumanyaev",
"email"=>"me@zzet.org",
"time"=>"2014-02-16T02:24:23+04:00"},
"committer"=>
{"name"=>"Andrey Kumanyaev",
"email"=>"me@zzet.org",
"time"=>"2014-02-16T02:24:23+04:00"},
"message"=>"Save 2. Indexing work. Search too\n"}}}>
Project.last.repository.as_indexed_json
Project Load (1.7ms) SELECT "projects".* FROM "projects" ORDER BY "projects"."id" DESC LIMIT 1
Namespace Load (4.8ms) SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."id" = $1 ORDER BY "namespaces"."id" ASC LIMIT 1 [["id", 3739]]
Namespace Load (0.9ms) SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."path" = 'zzet' LIMIT 1
Project Load (0.7ms) SELECT "projects".* FROM "projects" WHERE "projects"."namespace_id" = 3739 AND "projects"."path" = 'elasticsearch-git' LIMIT 1
# Long lines are stripped manually
=> {:blobs=>
[[{:type=>"blob",
:id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_.gitignore",
:rid=>4328,
:oid=>"d87d4be66f458acd52878902bbf1391732ad21e1",
:content=>
"*.gem\n*.rbc\n.bundle\n.config\n.yardoc\nGemfile.lock\nInstalledFiles\n_yardoc\ncoverage\ndoc/\nlib/bundler/man\npkg\nrdoc\nspec/reports\ntest/tmp\ntest/version_tmp\ntmp\n",
:commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
{:type=>"blob",
:id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_Gemfile",
:rid=>4328,
:oid=>"7322405f8f3ee5de24f7a727940ac52543e8954c",
:content=>
"source 'https://rubygems.org'\n\n# Specify your gem's dependencies in elasticsearch-git.gemspec\ngemspec\n\ngem 'elasticsearch-model', github: 'elasticsearch/elasticsearch-rails'\ngem 'elasticsearc....."
:commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
{:type=>"blob",
:id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_LICENSE.txt",
:rid=>4328,
:oid=>"f99909cd4ecb6f2ad08f8e55aac3a9fcd86a2bd2",
:content=>
"Copyright (c) 2014 Andrey Kumanyaev\n\nMIT License\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n\"Softw...."
:commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
{:type=>"blob",
:id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_README.md",
:rid=>4328,
:oid=>"8258d574dfc8040a5d003f06c6493e0033527f36",
:content=>
"# Elasticsearch::Git\n\nAttention: Pre-pre-pre beta code. Not production.\n\n[Elasticsearch](https://github.com/elasticsearch/elasticsearch-rails/tree/master/elasticsearch-model) integrations for g...."
:commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
{:type=>"blob",
:id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_Rakefile",
:rid=>4328,
:oid=>"29955274e0d42e164337c411ad9144e8ffd7e46e",
:content=>"require \"bundler/gem_tasks\"\n",
:commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
{:type=>"blob",
:id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_elasticsearch-git.gemspec",
:rid=>4328,
:oid=>"67762437568dda1bb98ec5eca8be7e4a5c8115a9",
:content=>
"# coding: utf-8\nlib = File.expand_path('../lib', __FILE__)\n$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)\nrequire 'elasticsearch/git/version'\n\nGem::Specification.new do |spec|\n spec..."
:commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
{:type=>"blob",
:id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_elasticsearch/git.rb",
:rid=>4328,
:oid=>"d3817ec58af1f44dfd18856bf54ef2bf607901a8",
:content=>
"require \"elasticsearch/git/version\"\nrequire \"elasticsearch/git/model\"\nrequire \"elasticsearch/git/commit\"\n\nmodule Elasticsearch\n module Git\n class Test\n include Elasticsearch::..."
:commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
{:type=>"blob",
:id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_git/model.rb",
:rid=>4328,
:oid=>"3dfbae747f25391779fbe012fe8cc4f38cc4651c",
:content=>
"require 'active_support/concern'\nrequire 'active_model'\nrequire 'elasticsearch/model'\n\nmodule Elasticsearch\n module Git\n module Model\n extend ActiveSupport::Concern\n\n include..."
:commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
{:type=>"blob",
:id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_git/repository.rb",
:rid=>4328,
:oid=>"70fe59c8391f6c27adb79c3e45824e6b4cf9566c",
:content=>
"require 'active_support/concern'\nrequire 'active_model'\nrequire 'elasticsearch'\nrequire 'elasticsearch/model'\nrequire 'rugged'\nrequire 'gitlab_git'\n\nmodule Elasticsearch\n module Git\n m..."
:commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
{:type=>"blob",
:id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_git/version.rb",
:rid=>4328,
:oid=>"79e8082b122492464732f1fb43e9f2bdc96ea146",
:content=>
"module Elasticsearch\n module Git\n VERSION = \"0.0.1\"\n end\nend\n",
:commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
{:type=>"blob",
:id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_test/test_helper.rb",
:rid=>4328,
:oid=>"6acc0d2b7bf0f286557d3757c1140b41ab57e8f7",
:content=>
"require \"rubygems\"\nrequire 'bundler/setup'\nrequire 'pry'\n\nBundler.require\n\nrequire 'wrong/adapters/minitest'\n\nPROJECT_ROOT = File.join(Dir.pwd)\n\nWrong.config.color\n\nMinitest.autorun\n..."
:commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"}]],
:commits=>
[{:type=>"commit",
:sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28",
:author=>
{:name=>"Andrey Kumanyaev",
:email=>"me@zzet.org",
:time=>2014-02-16 13:50:32 +0400},
:committer=>
{:name=>"Andrey Kumanyaev",
:email=>"me@zzet.org",
:time=>2014-02-16 13:50:32 +0400},
:message=>"Improve readme\n"},
{:type=>"commit",
:sha=>"37f1b0710eb7f41254ae0c33db09794a25bbb246",
:author=>
{:name=>"Andrey Kumanyaev",
:email=>"me@zzet.org",
:time=>2014-02-16 13:49:25 +0400},
:committer=>
{:name=>"Andrey Kumanyaev",
:email=>"me@zzet.org",
:time=>2014-02-16 13:49:25 +0400},
:message=>"prepare first test release\n"},
{:type=>"commit",
:sha=>"1802bafa70d3b1678cfa46a482fd396dd8a4bd40",
:author=>
{:name=>"Andrey Kumanyaev",
:email=>"me@zzet.org",
:time=>2014-02-16 02:24:23 +0400},
:committer=>
{:name=>"Andrey Kumanyaev",
:email=>"me@zzet.org",
:time=>2014-02-16 02:24:23 +0400},
:message=>"Save 2. Indexing work. Search too\n"},
{:type=>"commit",
:sha=>"3ed383bfbf6cba611d191dbc3590779c0444b7f0",
:author=>
{:name=>"Andrey Kumanyaev",
:email=>"me@zzet.org",
:time=>2014-02-16 00:23:10 +0400},
:committer=>
{:name=>"Andrey Kumanyaev",
:email=>"me@zzet.org",
:time=>2014-02-16 00:23:10 +0400},
:message=>"Save commit\n"},
{:type=>"commit",
:sha=>"7021addf520a19bdeceef29947c8687965c132ff",
:author=>
{:name=>"Andrey Kumanyaev",
:email=>"me@zzet.org",
:time=>2014-02-15 14:28:43 +0400},
:committer=>
{:name=>"Andrey Kumanyaev",
:email=>"me@zzet.org",
:time=>2014-02-15 14:28:43 +0400},
:message=>"first commit\n"}]}
* Add Exceptions handlers for indexing (Error connections and timeouts)
git checkout -b my-new-feature)git commit -am 'Add some feature')git push origin my-new-feature)FAQs
Unknown package
We found that elasticsearch-git demonstrated a not healthy version release cadence and project activity because the last version was released a year ago. It has 2 open source maintainers collaborating on the project.
Did you know?
Socket for GitHub automatically highlights issues in each pull request and monitors the health of all your open source dependencies. Discover the contents of your packages and block harmful activity before you install or update your dependencies.
Security News
In this segment of the Risky Business podcast, Feross Aboukhadijeh and Patrick Gray discuss the challenges of tracking malware discovered in open source softare.
Research
Security News
A threat actor's playbook for exploiting the npm ecosystem was exposed on the dark web, detailing how to build a blockchain-powered botnet.
Security News
NVD’s backlog surpasses 20,000 CVEs as analysis slows and NIST announces new system updates to address ongoing delays.