[interchange-cvs] interchange - heins modified 5 files
interchange-core@interchange.redhat.com
interchange-core@interchange.redhat.com
Sat Dec 29 14:50:00 2001
User: heins
Date: 2001-12-29 19:49:33 GMT
Modified: lib/Vend Data.pm DbSearch.pm Glimpse.pm TextSearch.pm
Modified: lib/Vend/Table Common.pm
Log:
* Add new MIRROR parameter for databases. This allows creation of a
DBM or Memory mirror.
Sequence is:
1. Table is always imported at config time.
2. Table is only imported once.
3. Text file is not required, but should be set so export
can work properly and so there is a placeholder.
A complete configuration is:
Database products_mem products_mem.txt TAB
Database products_mem MIRROR products
Database products_mem MEMORY 1
MIRROR name is not set in stone yet, probably not apropos
enough; suggestions are welcome.
Benchmarks for 1000 separate (i.e. tag_data, no caching)
accesses (repeated tries show these are representative):
Doing products_mem:
Class=MEMORY DSN=
Data returned 247 bytes
TIME=0.200
Doing products:
Class=DBI DSN=dbi:mysql:test_mfound
Data returned 247 bytes
TIME=0.670
Doing alias_mem:
Class=MEMORY DSN=
Data returned 9 bytes
TIME=0.180
Doing alias:
Class=DBI DSN=dbi:Pg:dbname=wikicms
Data returned 9 bytes
TIME=0.700
Doing wiki_mem:
Class=GDBM DSN=
Data returned 400 bytes
TIME=0.300
Doing wiki_mem2:
Class=MEMORY DSN=
Data returned 400 bytes
TIME=0.180
Doing wiki:
Class=DBI DSN=dbi:Pg:dbname=wikicms
Data returned 400 bytes
TIME=0.710
Gains should be even bigger as no database connections are needed
on a busy system.
Revision Changes Path
2.5 +49 -3 interchange/lib/Vend/Data.pm
rev 2.5, prev_rev 2.4
Index: Data.pm
===================================================================
RCS file: /anon_cvs/repository/interchange/lib/Vend/Data.pm,v
retrieving revision 2.4
retrieving revision 2.5
diff -u -r2.4 -r2.5
--- Data.pm 2001/12/28 17:55:44 2.4
+++ Data.pm 2001/12/29 19:49:33 2.5
@@ -1,6 +1,6 @@
# Vend::Data - Interchange databases
#
-# $Id: Data.pm,v 2.4 2001/12/28 17:55:44 mheins Exp $
+# $Id: Data.pm,v 2.5 2001/12/29 19:49:33 mheins Exp $
#
# Copyright (C) 1996-2001 Red Hat, Inc. <interchange@redhat.com>
#
@@ -647,7 +647,22 @@
if($Global::Database) {
copyref($Global::Database, $Vend::Cfg->{Database});
}
- while (($name,$data) = each %{$Vend::Cfg->{Database}}) {
+
+ my @tables = keys %{$Vend::Cfg->{Database}};
+
+ my @delayed;
+ my $redone;
+
+ TIEDB: {
+
+ foreach $name (@tables) {
+ $data = $Vend::Cfg->{Database}{$name} || {};
+ if(! $redone and $data->{MIRROR}) {
+#::logDebug("mirror database $name, delaying");
+ $data->{HOT} = 1;
+ push @delayed, $name;
+ next;
+ }
if(! $data->{name}) {
#::logDebug("Screwed up database: " . ::uneval( $data) );
next;
@@ -679,6 +694,16 @@
$Vend::Database{$name} = new $class ($data);
}
}
+
+ # So mirrors will not happen until after mirror source
+ if(@delayed) {
+ @tables = @delayed;
+ @delayed = ();
+ $redone = 1;
+ redo TIEDB;
+ }
+
+ }
update_productbase();
}
@@ -816,6 +841,17 @@
}
}
+ if($obj->{MIRROR}) {
+ if($obj->{Mirror_complete}) {
+ $no_import = 1;
+ }
+ else {
+#::logDebug ("table $new_table_name: undeffing $database_dbm, hot=$obj->{HOT}");
+ undef $database_dbm;
+ undef $no_import;
+ }
+ }
+
last IMPORT if $no_import;
#::logDebug ("moving to import") if $type == 9;
@@ -826,6 +862,7 @@
if (
! defined $database_dbm
or ! -e $database_dbm
+ or $obj->{MIRROR}
or ($txt_time = file_modification_time($database_txt, $obj->{PRELOAD}))
>
($dbm_time = file_modification_time($database_dbm))
@@ -848,11 +885,20 @@
local($/) = $record_delim if defined $record_delim;
+ if($obj->{MIRROR}) {
+ $db = Vend::Table::Common::import_from_ic_db(
+ $database_txt,
+ $obj,
+ $new_table_name,
+ );
+ }
+ else {
$db = Vend::Table::Common::import_ascii_delimited(
$database_txt,
$obj,
$new_table_name,
);
+ }
$/ = $save;
if(defined $database_dbm) {
@@ -1021,7 +1067,7 @@
$scan =~ s:^scan/::;
my $c = {
- mv_list_only => 1,
+ mv_list_only => 1,
mv_search_file => $bx_fn,
};
2.6 +8 -8 interchange/lib/Vend/DbSearch.pm
rev 2.6, prev_rev 2.5
Index: DbSearch.pm
===================================================================
RCS file: /anon_cvs/repository/interchange/lib/Vend/DbSearch.pm,v
retrieving revision 2.5
retrieving revision 2.6
diff -u -r2.5 -r2.6
--- DbSearch.pm 2001/11/26 18:34:02 2.5
+++ DbSearch.pm 2001/12/29 19:49:33 2.6
@@ -1,6 +1,6 @@
# Vend::DbSearch - Search indexes with Interchange
#
-# $Id: DbSearch.pm,v 2.5 2001/11/26 18:34:02 mheins Exp $
+# $Id: DbSearch.pm,v 2.6 2001/12/29 19:49:33 mheins Exp $
#
# Adapted for use with Interchange from Search::TextSearch
#
@@ -26,7 +26,7 @@
@ISA = qw(Vend::Search);
-$VERSION = substr(q$Revision: 2.5 $, 10);
+$VERSION = substr(q$Revision: 2.6 $, 10);
use Search::Dict;
use strict;
@@ -34,24 +34,24 @@
sub array {
my ($s, $opt) = @_;
$s->{mv_one_sql_table} = 1;
- $s->{mv_list_only} = 1;
- Vend::Scan::perform_search($opt, undef, $s);
+ $s->{mv_list_only} = 1; # makes perform_search only return results array
+ return Vend::Scan::perform_search($opt, undef, $s);
}
sub hash {
my ($s, $opt) = @_;
$s->{mv_return_reference} = 'HASH';
$s->{mv_one_sql_table} = 1;
- $s->{mv_list_only} = 1;
- Vend::Scan::perform_search($opt, undef, $s);
+ $s->{mv_list_only} = 1; # makes perform_search only return results array
+ return Vend::Scan::perform_search($opt, undef, $s);
}
sub list {
my ($s, $opt) = @_;
- $s->{mv_list_only} = 1;
$s->{mv_return_reference} = 'LIST';
$s->{mv_one_sql_table} = 1;
- Vend::Scan::perform_search($opt, undef, $s);
+ $s->{mv_list_only} = 1; # makes perform_search only return results array
+ return Vend::Scan::perform_search($opt, undef, $s);
}
my %Default = (
2.3 +8 -8 interchange/lib/Vend/Glimpse.pm
rev 2.3, prev_rev 2.2
Index: Glimpse.pm
===================================================================
RCS file: /anon_cvs/repository/interchange/lib/Vend/Glimpse.pm,v
retrieving revision 2.2
retrieving revision 2.3
diff -u -r2.2 -r2.3
--- Glimpse.pm 2001/11/26 18:34:02 2.2
+++ Glimpse.pm 2001/12/29 19:49:33 2.3
@@ -1,6 +1,6 @@
# Vend::Glimpse - Search indexes with Glimpse
#
-# $Id: Glimpse.pm,v 2.2 2001/11/26 18:34:02 mheins Exp $
+# $Id: Glimpse.pm,v 2.3 2001/12/29 19:49:33 mheins Exp $
#
# Adapted for use with Interchange from Search::Glimpse
#
@@ -25,27 +25,27 @@
require Vend::Search;
@ISA = qw(Vend::Search);
-$VERSION = substr(q$Revision: 2.2 $, 10);
+$VERSION = substr(q$Revision: 2.3 $, 10);
use strict;
sub array {
my ($s, $opt) = @_;
- $s->{mv_list_only} = 1;
- Vend::Scan::perform_search($opt, undef, $s);
+ $s->{mv_list_only} = 1; # makes perform_search only return results array
+ return Vend::Scan::perform_search($opt, undef, $s);
}
sub hash {
my ($s, $opt) = @_;
$s->{mv_return_reference} = 'HASH';
- $s->{mv_list_only} = 1;
- Vend::Scan::perform_search($opt, undef, $s);
+ $s->{mv_list_only} = 1; # makes perform_search only return results array
+ return Vend::Scan::perform_search($opt, undef, $s);
}
sub list {
my ($s, $opt) = @_;
- $s->{mv_list_only} = 1;
$s->{mv_return_reference} = 'LIST';
- Vend::Scan::perform_search($opt, undef, $s);
+ $s->{mv_list_only} = 1; # makes perform_search only return results array
+ return Vend::Scan::perform_search($opt, undef, $s);
}
my %Default = (
2.3 +8 -8 interchange/lib/Vend/TextSearch.pm
rev 2.3, prev_rev 2.2
Index: TextSearch.pm
===================================================================
RCS file: /anon_cvs/repository/interchange/lib/Vend/TextSearch.pm,v
retrieving revision 2.2
retrieving revision 2.3
diff -u -r2.2 -r2.3
--- TextSearch.pm 2001/12/07 04:16:06 2.2
+++ TextSearch.pm 2001/12/29 19:49:33 2.3
@@ -1,6 +1,6 @@
# Vend::TextSearch - Search indexes with Perl
#
-# $Id: TextSearch.pm,v 2.2 2001/12/07 04:16:06 jon Exp $
+# $Id: TextSearch.pm,v 2.3 2001/12/29 19:49:33 mheins Exp $
#
# Adapted for use with Interchange from Search::TextSearch
#
@@ -28,29 +28,29 @@
use vars qw(@ISA);
@ISA = qw(Vend::Search);
-$VERSION = substr(q$Revision: 2.2 $, 10);
+$VERSION = substr(q$Revision: 2.3 $, 10);
use Search::Dict;
use strict;
sub array {
my ($s, $opt) = @_;
- $s->{mv_list_only} = 1;
- Vend::Scan::perform_search($opt, undef, $s);
+ $s->{mv_list_only} = 1; # makes perform_search only return results array
+ return Vend::Scan::perform_search($opt, undef, $s);
}
sub hash {
my ($s, $opt) = @_;
$s->{mv_return_reference} = 'HASH';
- $s->{mv_list_only} = 1;
- Vend::Scan::perform_search($opt, undef, $s);
+ $s->{mv_list_only} = 1; # makes perform_search only return results array
+ return Vend::Scan::perform_search($opt, undef, $s);
}
sub list {
my ($s, $opt) = @_;
- $s->{mv_list_only} = 1;
$s->{mv_return_reference} = 'LIST';
- Vend::Scan::perform_search($opt, undef, $s);
+ $s->{mv_list_only} = 1; # makes perform_search only return results array
+ return Vend::Scan::perform_search($opt, undef, $s);
}
my %Default = (
2.8 +62 -2 interchange/lib/Vend/Table/Common.pm
rev 2.8, prev_rev 2.7
Index: Common.pm
===================================================================
RCS file: /anon_cvs/repository/interchange/lib/Vend/Table/Common.pm,v
retrieving revision 2.7
retrieving revision 2.8
diff -u -r2.7 -r2.8
--- Common.pm 2001/12/07 00:20:28 2.7
+++ Common.pm 2001/12/29 19:49:33 2.8
@@ -1,6 +1,6 @@
# Vend::Table::Common - Common access methods for Interchange databases
#
-# $Id: Common.pm,v 2.7 2001/12/07 00:20:28 jon Exp $
+# $Id: Common.pm,v 2.8 2001/12/29 19:49:33 mheins Exp $
#
# Copyright (C) 1996-2001 Red Hat, Inc. <interchange@redhat.com>
#
@@ -22,7 +22,7 @@
# Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307 USA.
-$VERSION = substr(q$Revision: 2.7 $, 10);
+$VERSION = substr(q$Revision: 2.8 $, 10);
use strict;
package Vend::Table::Common;
@@ -1277,6 +1277,66 @@
);
}
return $out;
+}
+
+sub import_from_ic_db {
+ my ($infile, $options, $table_name) = @_;
+
+ my $tname = $options->{MIRROR}
+ or die errmsg(
+ "Memory mirror table not specified for table %s.",
+ $table_name,
+ );
+#::logDebug("Importing mirrored $table_name from $tname");
+
+ $Vend::Database{$tname} =
+ Vend::Data::import_database($Vend::Cfg->{Database}{$tname})
+ unless $Vend::Database{$tname};
+
+ my $idb = Vend::Data::database_exists_ref($tname)
+ or die errmsg(
+ "Memory mirror table %s does not exist (yet) to create mirror %s.\n",
+ $tname,
+ $table_name,
+ );
+
+ my @field_names = $idb->columns;
+
+ my $odb;
+
+ if($options->{ObjectType}) {
+ no strict 'refs';
+ $odb = &{"$options->{ObjectType}::create"}(
+ $options->{ObjectType},
+ $options,
+ \@field_names,
+ $table_name,
+ );
+ }
+ else {
+ $odb = $options->{Object};
+ }
+
+#::logDebug("idb=$idb odb=$odb");
+ eval {
+ my $f;
+ while($f = $idb->each_nokey()) {
+#::logDebug("importing key=$f->[0]");
+ $odb->set_row(@$f);
+ }
+ };
+
+ if($@) {
+ die ::errmsg(
+ "Problem with mirror import from source %s to target %s\n",
+ $tname,
+ $table_name,
+ );
+ }
+
+ $odb->[$CONFIG]{Mirror_complete} = 1;
+ delete $odb->[$CONFIG]{Clean_start};
+ return $odb;
}
my $white = ' \t';