Move searchdb inside app/upload/plugins/xapian/searchdb

1.10.x
Julio Montoya 11 years ago
parent 7127f6c0ac
commit f36a285e11
  1. 18
      .gitignore
  2. 2
      .travis.yml
  3. 2
      README.md
  4. 0
      app/upload/plugins/xapian/searchdb/index.html
  5. 2
      main/admin/settings.lib.php
  6. 19
      main/inc/lib/search/IndexableChunk.class.php
  7. 83
      main/inc/lib/search/xapian/XapianIndexer.class.php
  8. 2
      main/inc/lib/search/xapian/XapianQuery.php
  9. 2
      main/install/update-files-1.9.0-1.10.0.inc.php
  10. 8
      main/search/INSTALL
  11. 4
      robots.txt

18
.gitignore vendored

@ -17,28 +17,14 @@ courses/*
!courses/index.html !courses/index.html
# Home # Home
app/home/*
home/* home/*
# User images # Upload content
app/upload/users/* app/upload/*
!app/upload/users/index.html
# Session images
app/upload/sessions/*
!main/upload/sessions/index.html
# Course images
app/upload/courses/*
!app/upload/courses/index.html
# Logs and databases # # Logs and databases #
*.log *.log
# Xapian indexes directory
searchdb/*
!searchdb/index.html
# IDE settings # IDE settings
.idea .idea

@ -40,7 +40,7 @@ before_script:
- cp chamilo-cli-install/chamilo-cli-installer.php main/install/ - cp chamilo-cli-install/chamilo-cli-installer.php main/install/
- mysql -u root -e 'create database chamilo' - mysql -u root -e 'create database chamilo'
# install Chamilo with Chash - see reference https://github.com/sonnym/travis-ci-drupal-module-example/blob/master/.travis.yml # install Chamilo with Chash - see reference https://github.com/sonnym/travis-ci-drupal-module-example/blob/master/.travis.yml
- sudo chmod -R 0777 app/cache courses home app/upload/ main/default_course_document/images main/inc/conf searchdb main/lang main/css - sudo chmod -R 0777 app/config app/cache app/courses home app/upload/ main/default_course_document/images main/lang main/css
- cd main/install/ - cd main/install/
- sudo php5 chamilo-cli-installer.php -l admin -p admin -U travis -u 'http://localhost/' -X travis -L english -z 'admin@example.com' -f 'John' -g 'Doe' -b '555-5555' -c 'Test campus' -y 'Chamilo' -x 'https://chamilo.org' - sudo php5 chamilo-cli-installer.php -l admin -p admin -U travis -u 'http://localhost/' -X travis -L english -z 'admin@example.com' -f 'John' -g 'Doe' -b '555-5555' -c 'Test campus' -y 'Chamilo' -x 'https://chamilo.org'
- cd ../.. - cd ../..

@ -59,7 +59,7 @@ composer update
On a Debian-based system, launch: On a Debian-based system, launch:
``` ```
sudo chown -R www-data:www-data app/cache course home searchdb app/upload/users app/upload/sessions app/upload/courses main/default_course_document/images main/lang main/css main/inc/conf sudo chown -R www-data:www-data app/cache app/config app/course home app/upload main/default_course_document/images main/lang main/css
``` ```
### Start the installer ### Start the installer

@ -656,7 +656,7 @@ function handle_search()
echo '</div>'; echo '</div>';
if ($search_enabled == 'true') { if ($search_enabled == 'true') {
$xapian_path = api_get_path(SYS_PATH).'searchdb'; $xapian_path = api_get_path(SYS_UPLOAD_PATH).'plugins/xapian/searchdb';
/* /*
@todo Test the Xapian connection @todo Test the Xapian connection

@ -1,12 +1,10 @@
<?php <?php
/* For licensing terms, see /license.txt */ /* For licensing terms, see /license.txt */
/** /**
* @package chamilo.include.search * @package chamilo.include.search
*/ */
/**
* Code
*/
// some constants to avoid serialize string keys on serialized data array // some constants to avoid serialize string keys on serialized data array
define('SE_COURSE_ID', 0); define('SE_COURSE_ID', 0);
define('SE_TOOL_ID', 1); define('SE_TOOL_ID', 1);
@ -25,7 +23,8 @@ define('XAPIAN_PREFIX_TOOLID', 'O');
* Class * Class
* @package chamilo.include.search * @package chamilo.include.search
*/ */
abstract class _IndexableChunk { abstract class _IndexableChunk
{
/* struct (array) /* struct (array)
* { * {
* string title; <- nombre de archivo/elemento * string title; <- nombre de archivo/elemento
@ -97,20 +96,22 @@ abstract class _IndexableChunk {
* Extension of the _IndexableChunk class to make IndexableChunk extensible. * Extension of the _IndexableChunk class to make IndexableChunk extensible.
* @package chamilo.include.search * @package chamilo.include.search
*/ */
class IndexableChunk extends _IndexableChunk { class IndexableChunk extends _IndexableChunk
{
/** /**
* Let add course id term * Let add course id term
*/ */
public function addCourseId($course_id) { public function addCourseId($course_id)
{
$this->addTerm($course_id, XAPIAN_PREFIX_COURSEID); $this->addTerm($course_id, XAPIAN_PREFIX_COURSEID);
} }
/** /**
* Let add tool id term * Let add tool id term
*/ */
public function addToolId($tool_id) { public function addToolId($tool_id)
{
$this->addTerm($tool_id, XAPIAN_PREFIX_TOOLID); $this->addTerm($tool_id, XAPIAN_PREFIX_TOOLID);
} }
} }

@ -1,12 +1,9 @@
<?php <?php
/* For licensing terms, see /license.txt */ /* For licensing terms, see /license.txt */
/** /**
* @package chamilo.include.search * @package chamilo.include.search
*/ */
/**
* Code
*/
require_once 'xapian.php'; require_once 'xapian.php';
require_once dirname(__FILE__) . '/../IndexableChunk.class.php'; require_once dirname(__FILE__) . '/../IndexableChunk.class.php';
@ -15,7 +12,8 @@ require_once dirname(__FILE__) . '/../IndexableChunk.class.php';
* Abstract helper class * Abstract helper class
* @package chamilo.include.search * @package chamilo.include.search
*/ */
abstract class XapianIndexer { abstract class XapianIndexer
{
/* XapianWritableDatabase */ /* XapianWritableDatabase */
protected $db; protected $db;
@ -26,6 +24,15 @@ abstract class XapianIndexer {
/* XapianStem */ /* XapianStem */
public $stemmer; public $stemmer;
/**
* Class contructor
*/
function __construct()
{
$this->db = null;
$this->stemmer = null;
}
/** /**
* Generates a list of languages Xapian manages * Generates a list of languages Xapian manages
* *
@ -33,7 +40,8 @@ abstract class XapianIndexer {
* Chamilo languages and Xapian languages (through hardcoding) * Chamilo languages and Xapian languages (through hardcoding)
* @return array Array of languages codes -> Xapian languages * @return array Array of languages codes -> Xapian languages
*/ */
public final function xapian_languages() { public final function xapian_languages()
{
/* http://xapian.org/docs/apidoc/html/classXapian_1_1Stem.html */ /* http://xapian.org/docs/apidoc/html/classXapian_1_1Stem.html */
return array( return array(
'none' => 'none', //don't stem terms 'none' => 'none', //don't stem terms
@ -60,14 +68,15 @@ abstract class XapianIndexer {
/** /**
* Connect to the database, and create it if it doesn't exist * Connect to the database, and create it if it doesn't exist
*/ */
function connectDb($path = NULL, $dbMode = NULL, $lang = 'english') { function connectDb($path = null, $dbMode = null, $lang = 'english')
if ($this->db != NULL) {
if ($this->db != null)
return $this->db; return $this->db;
if ($dbMode == NULL) if ($dbMode == null)
$dbMode = Xapian::DB_CREATE_OR_OPEN; $dbMode = Xapian::DB_CREATE_OR_OPEN;
if ($path == NULL) if ($path == null)
$path = api_get_path(SYS_PATH) . 'searchdb/'; $path = api_get_path(SYS_UPLOAD_PATH).'plugins/xapian/searchdb/';
try { try {
$this->db = new XapianWritableDatabase($path, $dbMode); $this->db = new XapianWritableDatabase($path, $dbMode);
@ -82,6 +91,7 @@ abstract class XapianIndexer {
return $this->db; return $this->db;
} catch (Exception $e) { } catch (Exception $e) {
Display::display_error_message($e->getMessage()); Display::display_error_message($e->getMessage());
return 1; return 1;
} }
} }
@ -90,7 +100,8 @@ abstract class XapianIndexer {
* Simple getter for the db attribute * Simple getter for the db attribute
* @return object The db attribute * @return object The db attribute
*/ */
function getDb() { function getDb()
{
return $this->db; return $this->db;
} }
@ -99,16 +110,18 @@ abstract class XapianIndexer {
* @param string Chunk of text * @param string Chunk of text
* @return void * @return void
*/ */
function addChunk($chunk) { function addChunk($chunk)
{
$this->chunks[] = $chunk; $this->chunks[] = $chunk;
} }
/** /**
* Actually index the current data * Actually index the current data
* *
* @return integer New Xapian document ID or NULL upon failure * @return integer New Xapian document ID or null upon failure
*/ */
function index() { function index()
{
try { try {
if (!empty($this->chunks)) { if (!empty($this->chunks)) {
foreach ($this->chunks as $chunk) { foreach ($this->chunks as $chunk) {
@ -148,8 +161,9 @@ abstract class XapianIndexer {
* @param int did Xapian::docid * @param int did Xapian::docid
* @return mixed XapianDocument, or false on error * @return mixed XapianDocument, or false on error
*/ */
function get_document($did) { function get_document($did)
if ($this->db == NULL) { {
if ($this->db == null) {
$this->connectDb(); $this->connectDb();
} }
try { try {
@ -167,8 +181,9 @@ abstract class XapianIndexer {
* @param XapianDocument $doc xapian document to push into the db * @param XapianDocument $doc xapian document to push into the db
* @return mixed xapian document data or FALSE if error * @return mixed xapian document data or FALSE if error
*/ */
function get_document_data($doc) { function get_document_data($doc)
if ($this->db == NULL) { {
if ($this->db == null) {
$this->connectDb(); $this->connectDb();
} }
try { try {
@ -191,7 +206,8 @@ abstract class XapianIndexer {
* @param string $prefix Prefix used to categorize the doc (usually 'T' for title, 'A' for author) * @param string $prefix Prefix used to categorize the doc (usually 'T' for title, 'A' for author)
* @return boolean false on error * @return boolean false on error
*/ */
function update_terms($did, $terms, $prefix) { function update_terms($did, $terms, $prefix)
{
$doc = $this->get_document($did); $doc = $this->get_document($did);
if ($doc === false) { if ($doc === false) {
return false; return false;
@ -203,6 +219,7 @@ abstract class XapianIndexer {
} }
$this->db->replace_document($did, $doc); $this->db->replace_document($did, $doc);
$this->db->flush(); $this->db->flush();
return true; return true;
} }
@ -211,8 +228,9 @@ abstract class XapianIndexer {
* *
* @param int did Xapian::docid * @param int did Xapian::docid
*/ */
function remove_document($did) { function remove_document($did)
if ($this->db == NULL) { {
if ($this->db == null) {
$this->connectDb(); $this->connectDb();
} }
if (is_numeric($did) && $did > 0) { if (is_numeric($did) && $did > 0) {
@ -231,7 +249,8 @@ abstract class XapianIndexer {
* @param XapianDocument $doc The xapian document where to add the term * @param XapianDocument $doc The xapian document where to add the term
* @return mixed XapianDocument, or false on error * @return mixed XapianDocument, or false on error
*/ */
function add_term_to_doc($term, $doc) { function add_term_to_doc($term, $doc)
{
if (!is_a($doc, 'XapianDocument')) { if (!is_a($doc, 'XapianDocument')) {
return FALSE; return FALSE;
} }
@ -250,7 +269,8 @@ abstract class XapianIndexer {
* @param XapianDocument $doc The xapian document where to add the term * @param XapianDocument $doc The xapian document where to add the term
* @return mixed XapianDocument, or false on error * @return mixed XapianDocument, or false on error
*/ */
function remove_term_from_doc($term, $doc) { function remove_term_from_doc($term, $doc)
{
if (!is_a($doc, 'XapianDocument')) { if (!is_a($doc, 'XapianDocument')) {
return FALSE; return FALSE;
} }
@ -268,11 +288,12 @@ abstract class XapianIndexer {
* @param XapianDocument $doc xapian document to push into the db * @param XapianDocument $doc xapian document to push into the db
* @param Xapian::docid $did xapian document id of the document to replace * @param Xapian::docid $did xapian document id of the document to replace
*/ */
function replace_document($doc, $did) { function replace_document($doc, $did)
{
if (!is_a($doc, 'XapianDocument')) { if (!is_a($doc, 'XapianDocument')) {
return FALSE; return FALSE;
} }
if ($this->db == NULL) { if ($this->db == null) {
$this->connectDb(); $this->connectDb();
} }
try { try {
@ -284,18 +305,12 @@ abstract class XapianIndexer {
} }
} }
/**
* Class contructor
*/
function __construct() {
$this->db = NULL;
$this->stemmer = NULL;
}
/** /**
* Class destructor * Class destructor
*/ */
function __destruct() { function __destruct()
{
unset($this->db); unset($this->db);
unset($this->stemmer); unset($this->stemmer);
} }

@ -12,7 +12,7 @@ require_once dirname(__FILE__) . '/../IndexableChunk.class.php';
//TODO: think another way without including specific fields here //TODO: think another way without including specific fields here
require_once api_get_path(LIBRARY_PATH) . 'specific_fields_manager.lib.php'; require_once api_get_path(LIBRARY_PATH) . 'specific_fields_manager.lib.php';
define('XAPIAN_DB', api_get_path(SYS_PATH) . 'searchdb/'); define('XAPIAN_DB', api_get_path(SYS_UPLOAD_PATH) . 'plugins/xapian/searchdb/');
/** /**
* Queries the database. * Queries the database.

@ -86,6 +86,7 @@ if (defined('SYSTEM_INSTALLATION')) {
'xhosa', 'xhosa',
'yoruba', 'yoruba',
); );
$filesToDelete = array( $filesToDelete = array(
'accessibility', 'accessibility',
'admin', 'admin',
@ -169,6 +170,7 @@ if (defined('SYSTEM_INSTALLATION')) {
api_get_path(SYS_CODE_PATH).'upload/users' => api_get_path(SYS_UPLOAD_PATH).'users', api_get_path(SYS_CODE_PATH).'upload/users' => api_get_path(SYS_UPLOAD_PATH).'users',
api_get_path(SYS_CODE_PATH).'upload/badges' => api_get_path(SYS_UPLOAD_PATH).'badges', api_get_path(SYS_CODE_PATH).'upload/badges' => api_get_path(SYS_UPLOAD_PATH).'badges',
api_get_path(SYS_PATH).'courses' => api_get_path(SYS_COURSE_PATH), api_get_path(SYS_PATH).'courses' => api_get_path(SYS_COURSE_PATH),
api_get_path(SYS_PATH).'searchdb' => api_get_path(SYS_UPLOAD_PATH).'plugins/xapian/searchdb',
]; ];
foreach ($movePathList as $origin => $destination) { foreach ($movePathList as $origin => $destination) {

@ -8,8 +8,8 @@ On Debian Lenny
http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=493944 http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=493944
http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=493941 http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=493941
on dokeos root: on dokeos root:
mkdir searchdb mkdir app/upload/plugins/xapian/searchdb
chmod 777 searchdb (or equivalent) chmod 777 app/upload/plugins/xapian/searchdb (or equivalent)
Useful xapian development tools Useful xapian development tools
apt-get install xapian-doc xapian-tools apt-get install xapian-doc xapian-tools
(See delve command) (See delve command)
@ -29,8 +29,8 @@ Chamilo 1.8.8 + XAPIAN in Ubuntu 10.10
sudo apt-get install imagemagick sudo apt-get install imagemagick
sudo apt-get install php5-imagick sudo apt-get install php5-imagick
6. In Chamilo root 6. In Chamilo root
mkdir searchdb mkdir app/upload/plugins/xapian/searchdb
sudo chmod 777 searchdb sudo chmod 777 app/upload/plugins/xapian/searchdb
7. Useful Xapian development tools 7. Useful Xapian development tools
sudo apt-get install xapian-doc xapian-tools sudo apt-get install xapian-doc xapian-tools
8. Restart Apache 8. Restart Apache

@ -18,14 +18,12 @@ User-Agent: *
# Directories # Directories
Disallow: /app/cache/ Disallow: /app/cache/
Disallow: /app/logs/ Disallow: /app/logs/
Disallow: /app/upload/
Disallow: /app/courses/ Disallow: /app/courses/
Disallow: /courses/
Disallow: /documentation/ Disallow: /documentation/
Disallow: /app/home/
Disallow: /home/ Disallow: /home/
Disallow: /main/ Disallow: /main/
Disallow: /plugin/ Disallow: /plugin/
Disallow: /searchdb/
Disallow: /tests/ Disallow: /tests/
# Files # Files

Loading…
Cancel
Save