From 04ee5ee28ad3820663483420092b4a8e00c2306b Mon Sep 17 00:00:00 2001 From: catherine <catherine@orlinresearch.com> Date: Sat, 16 May 2015 17:10:17 -0700 Subject: [PATCH] Get adapter to work with ruby 2 --- .../activerecord-monetdb-adapter-0.1.gemspec | 4 +- .../connection_adapters/monetdb_adapter.rb | 438 +++++++++--------- .../monetdb_adapter_config.rb | 13 - lib/MonetDB.rb | 3 +- lib/MonetDBConnection.rb | 250 +++++----- lib/MonetDBData.rb | 267 ++++++----- ruby-monetdb-sql-0.1.gemspec | 4 +- 7 files changed, 477 insertions(+), 502 deletions(-) delete mode 100644 adapter/lib/active_record/connection_adapters/monetdb_adapter_config.rb diff --git a/adapter/activerecord-monetdb-adapter-0.1.gemspec b/adapter/activerecord-monetdb-adapter-0.1.gemspec index 3a623cf..83d91d3 100644 --- a/adapter/activerecord-monetdb-adapter-0.1.gemspec +++ b/adapter/activerecord-monetdb-adapter-0.1.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| - s.required_ruby_version = '>= 1.8.0' + s.required_ruby_version = '>= 2.1.0' s.name = %q{activerecord-monetdb-adapter} - s.version = "0.1" + s.version = "0.2" s.date = %q{2009-05-18} s.authors = ["G Modena"] s.email = %q{gm@cwi.nl} diff --git a/adapter/lib/active_record/connection_adapters/monetdb_adapter.rb b/adapter/lib/active_record/connection_adapters/monetdb_adapter.rb index c11f015..fa5a799 100644 --- a/adapter/lib/active_record/connection_adapters/monetdb_adapter.rb +++ b/adapter/lib/active_record/connection_adapters/monetdb_adapter.rb @@ -23,7 +23,9 @@ # Refreshed by Martin Samson (2011) -require 'active_record/connection_adapters/monetdb_adapter_config' # added - I.T. +MDB_SYS_SCHEMA = "sys." +MDB_NON_SYSTEM_TABLES_ONLY = "and system = false" + require 'active_record/connection_adapters/abstract_adapter' require 'MonetDB' @@ -32,12 +34,12 @@ module ActiveRecord # Establishes a connection to the database that's used by all Active Record objects def self.monetdb_connection(config) # extract connection parameters - config = config.symbolize_keys + config = config.symbolize_keys host = config[:host] || "127.0.0.1" port = config[:port] || 50000 - username = config[:username].to_s if config[:username] - password = config[:password].to_s if config[:password] + username = config[:username].to_s if config[:username] + password = config[:password].to_s if config[:password] # Use "sql" as default language if none is specified lang = config[:lang] || "sql" @@ -54,7 +56,6 @@ module ActiveRecord end - module ConnectionAdapters class MonetDBColumn < Column # Handles the case where column type is int but default @@ -63,132 +64,130 @@ module ActiveRecord # schema_definitions does not return a fixnum(0 or 1) but # the correct default value. def type_cast(value) - if( value.nil? ) - return nil - elsif( type == :integer && value =~/next value for/ ) - #return value - return nil + if value.nil? + nil + elsif type == :integer && value =~/next value for/ + nil else super end end - + private - - def simplified_type(field_type) - case field_type - when /int|smallint/i - :integer - when /real|double/i - :float - when /datetime/i - :timestamp - when /timestamp/i - :timestamp - when /char/i, /varchar/i - :string - when /bigint/i - :bigint - else - super - end - end - + + def simplified_type(field_type) + case field_type + when /int|smallint/i + :integer + when /real|double/i + :float + when /datetime/i + :timestamp + when /timestamp/i + :timestamp + when /char/i, /varchar/i + :string + when /bigint/i + :bigint + else + super + end + end + end #end of MonetDBColumn class - + class TableDefinition # Override so that we handle the fact that MonetDB # doesn't support "limit" on integer column. # Otherwise same implementation def column(name, type, options = {}) column = self[name] || ColumnDefinition.new(@base, name, type) - - if(type.to_sym != :integer and type.to_sym != :primary_key) + + if type.to_sym != :integer and type.to_sym != :primary_key column.limit = options[:limit] || native[type.to_sym][:limit] if options[:limit] or native[type.to_sym] end - + column.precision = options[:precision] column.scale = options[:scale] column.default = options[:default] column.null = options[:null] - + @columns << column unless @columns.include? column self end - + end - + class MonetDBAdapter < AbstractAdapter class BindSubstitution < Arel::Visitors::MySQL include Arel::Visitors::BindVisitor end - def initialize(connection, logger, connection_options, config) + def initialize(connection, logger, connection_options, config) super(connection, logger) @visitor = BindSubstitution.new self @connection_options, @config = connection_options, config connect end - + def adapter_name #:nodoc: 'MonetDB' end - + # Functions like rename_table, rename_column and # change_column cannot be implemented in MonetDB. def supports_migrations? true end - + # testing savepoints in progress def supports_savepoints? #:nodoc: true end - + def support_transaction? #:nodoc: false end - + def supports_ddl_transactions? false end - + def native_database_types - { + { :primary_key => "int NOT NULL auto_increment PRIMARY KEY", - :string => { :name => "varchar", :limit => 255 }, - :text => { :name => "clob" }, - :integer => { :name => "int"}, - :float => { :name => "float" }, - :decimal => { :name => "decimal" }, - :datetime => { :name => "timestamp" }, - :timestamp => { :name => "timestamp" }, - :time => { :name => "time" }, - :date => { :name => "date" }, - :binary => { :name => "blob" }, - :boolean => { :name => "boolean" }, - :bigint => { :name => "bigint" } - } - end - - + :string => {:name => "varchar", :limit => 255}, + :text => {:name => "clob"}, + :integer => {:name => "int"}, + :float => {:name => "float"}, + :decimal => {:name => "decimal"}, + :datetime => {:name => "timestamp"}, + :timestamp => {:name => "timestamp"}, + :time => {:name => "time"}, + :date => {:name => "date"}, + :binary => {:name => "blob"}, + :boolean => {:name => "boolean"}, + :bigint => {:name => "bigint"} + } + end + #MonetDB does not support using DISTINCT withing COUNT #by default def supports_count_distinct? false end - + #----------CONNECTION MANAGEMENT------------------ - + # Check if the connection is active def active? if @connection != nil @connection.is_connected? end - + return false end - + # Close this connection and open a new one in its place. def reconnect! if @connection != nil @@ -196,29 +195,26 @@ module ActiveRecord false end end - + def disconnect! - #@connection.auto_commit(flag=true) - @connection.close + #@connection.auto_commit(flag=true) + @connection.close end - + # -------END OF CONNECTION MANAGEMENT---------------- - - - + + # ===============SCHEMA DEFINITIONS===========# - - def binary_to_string(value) + + def binary_to_string(value) res = "" value.scan(/../).each { |i| res << i.hex.chr } res end - + # ===========END OF SCHEMA DEFINITIONS========# - - - - + + #===============SCHEMA STATEMENTS===========# # The following schema_statements.rb functions are not supported by MonetDB (19/5/2008). # @@ -234,34 +230,33 @@ module ActiveRecord # NOTE WE MAY BE ABLE TO "CHANGE" A COLUMN DEFINITION IF WE DROP THE COLUMN # AND CREATE A NEW ONE WITH THE OLD NAME. THIS COULD WORK AS LONG AS WE DON'T # LOSE ANY DATA. - - - + + # Sets a new default value for a column. # ===== Examples ===== # change_column_default(:suppliers, :qualification, 'new') # change_column_default(:accounts, :authorized, 1) def change_column_default(table_name, column_name, default) sql = "ALTER TABLE #{table_name} ALTER COLUMN #{quote_column_name(column_name)} SET DEFAULT" #{quote(default)}" - - if( default.nil? || (default.casecmp("NULL")==0) ) + + if (default.nil? || (default.casecmp("NULL")==0)) sql << " NULL" else sql << quote(default) end p "SQL: " + sql + '\n' - hdl = execute(sql) + hdl = execute(sql) end - + def remove_index(table_name, options = {}) hdl = execute("DROP INDEX #{index_name(table_name, options)}") end - + # MonetDB does not support limits on certain data types # Limit is supported for the {char, varchar, clob, blob, time, timestamp} data types. def type_to_sql(type, limit = nil, precision = nil, scale = nil) return super if limit.nil? - + # strip off limits on data types not supporting them if [:integer, :double, :date, :bigint].include? type type.to_s @@ -269,21 +264,21 @@ module ActiveRecord super end end - + # Returns an array of Column objects for the table specified by +table_name+. - def columns(table_name, name = nil) + def columns(table_name, name = nil) return [] if table_name.to_s.strip.empty? table_name = table_name.to_s if table_name.is_a?(Symbol) table_name = table_name.split('.')[-1] unless table_name.nil? - hdl = execute(" SELECT name, type, type_digits, type_scale, \"default\", \"null\" FROM #{MDB_SYS_SCHEMA}_columns WHERE table_id in (SELECT id FROM #{MDB_SYS_SCHEMA}_tables WHERE name = '#{table_name}' #{MDB_NON_SYSTEM_TABLES_ONLY})" ,name) - + hdl = execute(" SELECT name, type, type_digits, type_scale, \"default\", \"null\" FROM #{MDB_SYS_SCHEMA}_columns WHERE table_id in (SELECT id FROM #{MDB_SYS_SCHEMA}_tables WHERE name = '#{table_name}' #{MDB_NON_SYSTEM_TABLES_ONLY})", name) + num_rows = hdl.num_rows return [] unless num_rows >= 1 - - result = [] - - + + result = [] + + while row = hdl.fetch_hash do col_name = row['name'] col_default = row['default'] @@ -291,82 +286,82 @@ module ActiveRecord col_default = nil if (col_default && col_default.upcase == 'NULL') # Removes single quotes from the default value col_default.gsub!(/^'(.*)'$/, '\1') unless col_default.nil? - - # A string is returned so we must convert it to boolean + + # A string is returned so we must convert it to boolean col_nullable = row['null'] - - if( col_nullable.casecmp("true") == 0 ) - col_nullable = true - elsif( col_nullable.casecmp("false") == 0 ) + + if (col_nullable.casecmp("true") == 0) + col_nullable = true + elsif (col_nullable.casecmp("false") == 0) col_nullable = false - end - + end + col_type = row['type'] type_digits = row['type_digits'] type_scale = row['type_scale'] - - + + # Don't care about datatypes that aren't supported by # ActiveRecord, like interval. # Also do nothing for datatypes that don't support limit # like integer, double, date, bigint - if(col_type == "clob" || col_type == "blob") - if(type_digits.to_i > 0) - col_type << "(#{type_digits})" - end - elsif(col_type == "char" || - col_type == "varchar" || - col_type == "time" || - col_type == "timestamp" - ) + if (col_type == "clob" || col_type == "blob") + if (type_digits.to_i > 0) + col_type << "(#{type_digits})" + end + elsif (col_type == "char" || + col_type == "varchar" || + col_type == "time" || + col_type == "timestamp" + ) col_type << "(#{type_digits})" - elsif(col_type == "decimal") - if(type_scale.to_i == 0) + elsif (col_type == "decimal") + if (type_scale.to_i == 0) col_type << "(#{type_digits})" else col_type << "(#{type_digits},#{type_scale})" end end - + # instantiate a new column and insert into the result array result << MonetDBColumn.new(col_name, col_default, col_type, col_nullable) - + end - + # check that free has been correctly performed hdl.free - + return result end def primary_key(table) 'id' end - + # Adds a new column to the named table. # See TableDefinition#column for details of the options you can use. def add_column(table_name, column_name, type, options = {}) - if( (type.to_sym == :decimal) && (options[:precision].to_i+options[:scale].to_i > 18) ) + if ((type.to_sym == :decimal) && (options[:precision].to_i+options[:scale].to_i > 18)) raise StandardError, "It is not possible to have a decimal column where Precision + Scale > 18 . The column will not be added to the table!" - return + return else - super + super end - end - + end + # Return an array with all non-system table names of the current # database schema def tables(name = nil) - cur_schema = select_value("select current_schema", name) + cur_schema = select_value("select current_schema", name) select_values(" SELECT t.name FROM #{MDB_SYS_SCHEMA}_tables t, sys.schemas s WHERE s.name = '#{cur_schema}' AND t.schema_id = s.id - AND t.system = false",name) + AND t.system = false", name) end - + # Returns an array of indexes for the given table. def indexes(table_name, name = nil) - sql_query = " SELECT distinct i.name as index_name, k.\"name\", k.nr + sql_query = " SELECT distinct i.name as index_name, k.\"name\", k.nr FROM #{MDB_SYS_SCHEMA}idxs i, #{MDB_SYS_SCHEMA}_tables t, #{MDB_SYS_SCHEMA}objects k WHERE @@ -374,25 +369,25 @@ module ActiveRecord AND i.id = k.id AND t.id = i.table_id AND t.name = '#{table_name.to_s}' ORDER BY i.name, k.nr;" - result = select_all(sql_query,name); - + result = select_all(sql_query, name); + cur_index = nil indexes = [] - + result.each do |row| - if cur_index != row['index_name'] + if cur_index != row['index_name'] indexes << IndexDefinition.new(table_name, row['index_name'], false, []) cur_index = row['index_name'] - end - + end + indexes.last.columns << row['name'] end - - indexes + + indexes end - + # ===========END OF SCHEMA STATEMENTS========# - + # ===========QUOTING=========================# def quote(value, column = nil) if value.kind_of?(String) && column && column.type == :binary && column.class.respond_to?(:string_to_binary) @@ -402,48 +397,45 @@ module ActiveRecord super end end - + def quote_column_name(name) #:nodoc: "\"#{name.to_s}\"" end - + def quote_table_name(name) #:nodoc: quote_column_name(name).gsub('.', '"."') end - + # If the quoted true is 'true' MonetDB throws a string cast exception def quoted_true "true" end - + # If the quoted false is 'false' MonetDB throws a string cast exception def quoted_false "false" end - + # ===========END-OF-QUOTING==================# - - + + # =========DATABASE=STATEMENTS===============# - + # Returns an array of arrays containing the field values. # Order of columns in tuple arrays is not guaranteed. def select_rows(sql, name = nil) result = select(sql, name) - result.map{ |v| v.values} + result.map { |v| v.values } end - + def execute(sql, name = nil) - #puts "execute: #{sql}" - # This substitution is needed. - sql = sql.gsub('!=', '<>') + sql = sql.gsub('!=', '<>') sql += ';' @connection.query(sql) end def exec_query(sql, name = nil, binds = []) - #puts "exec_query: #{sql}" - @connection.query(sql) + select_rows(sql, name) end def last_inserted_id(result) @@ -454,76 +446,76 @@ module ActiveRecord res = super(arel, name, binds) res.affected_rows end - + # Begins the transaction. def begin_db_transaction hdl = execute("START TRANSACTION") end - + # Commits the transaction (ends TRANSACTIOM). def commit_db_transaction hdl = execute("COMMIT") end - + # Rolls back the transaction. Must be # done if the transaction block raises an exception or returns false (ends TRANSACTIOM). def rollback_db_transaction hdl = execute("ROLLBACK") end - + def current_savepoint_name @connection.transactions || 0 end - + # Create a new savepoint def create_savepoint - @connection.save - execute("SAVEPOINT #{current_savepoint_name}") + @connection.save + execute("SAVEPOINT #{current_savepoint_name}") end - + # rollback to the last savepoint def rollback_to_savepoint execute("ROLLBACK TO SAVEPOINT #{current_savepoint_name}") end - + # release current savepoint def release_savepoint execute("RELEASE SAVEPOINT #{current_savepoint_name}") end - - + def add_lock!(sql, options) @logger.info "Warning: MonetDB :lock option '#{options[:lock].inspect}' not supported. Returning unmodified sql statement!" if @logger && options.has_key?(:lock) sql end - + def empty_insert_statement(table_name) - # Ensures that the auto-generated id value will not violate the primary key constraint. - # comment out for production code(?) - #make_sure_pk_works(table_name, nil) + # Ensures that the auto-generated id value will not violate the primary key constraint. + # comment out for production code(?) + #make_sure_pk_works(table_name, nil) #"INSERT INTO #{quote_table_name(table_name)}" end - #=======END=OF=DATABASE=STATEMENTS=========# + + #=======END=OF=DATABASE=STATEMENTS=========# # Returns an array of record hashes with the column names as keys and # column values as values. def select(sql, name = nil, binds = []) - hdl = execute(sql,name) + hdl = execute(sql, name) hdl.result_hashes end # Executes the update statement and returns the number of rows affected. def update_sql(sql, name = nil) - hdl = execute(sql,name) + hdl = execute(sql, name) hdl.affected_rows end # Returns the last auto-generated ID from the affected table. def insert_sql(sql, name = nil, pk = nil, id_value = nil, sequence_name = nil) - # Ensures that the auto-generated id value will not violate the - # primary key constraint. Read the comments of make_sure_pk_works - # and documentation for further information. - # comment out for production code(?) + # Ensures that the auto-generated id value will not violate the + # primary key constraint. Read the comments of make_sure_pk_works + # and documentation for further information. + # comment out for production code(?) # table_name = extract_table_name_from_insertion_query(sql) # make_sure_pk_works(table_name,name) hdl = execute(sql, name) @@ -531,53 +523,53 @@ module ActiveRecord end protected - # Some tests insert some tuples with the id values set. In other words, the sequence - # is not used to generate a value for the primary key column named id. When a new tuple - # it to be inserted, where the id value is not set explicitly, a primary key violation will - # be raised because the generated from the sequence value is the same as one of the existing - # id values. This happens in unit tests quite often. So this function serves that the unit tests - # pass. However it is very expensive( sends 4 queries to the server) and probably not suitable for - # production code. Check the implementation for further info/details. - def make_sure_pk_works(table_name,name) - # Ensure the auto-generated id will not violate the primary key constraint. + # Some tests insert some tuples with the id values set. In other words, the sequence + # is not used to generate a value for the primary key column named id. When a new tuple + # it to be inserted, where the id value is not set explicitly, a primary key violation will + # be raised because the generated from the sequence value is the same as one of the existing + # id values. This happens in unit tests quite often. So this function serves that the unit tests + # pass. However it is very expensive( sends 4 queries to the server) and probably not suitable for + # production code. Check the implementation for further info/details. + def make_sure_pk_works(table_name, name) + # Ensure the auto-generated id will not violate the primary key constraint. # This is expensive and it's used so that the tests pass. Comment out for production code(?). - # Assume that table name has one primary key column named id that is associated with a sequence, - # otherwise return - hdl = nil - sequence_name = extract_sequence_name( select_value("select \"default\" from #{MDB_SYS_SCHEMA}_columns where table_id in (select id from #{MDB_SYS_SCHEMA}_tables where name = '#{table_name}') and name='id';") ) - - return if sequence_name.blank? - - max_id = select_value("select max(id) from #{table_name}").to_i - next_seq_val = select_value("select next value for #{sequence_name}").to_i - - if( max_id > next_seq_val ) - hdl = execute("ALTER SEQUENCE #{sequence_name} RESTART WITH #{max_id+1}" ,name) - else - hdl = execute("ALTER SEQUENCE #{sequence_name} RESTART WITH #{next_seq_val+1}",name) - end - end - - # Auxiliary function that extracts the table name from an insertion query - # It's called by insert_sql in order to assist at make_sure_pk_works. - # Ideally, if make_sure_pk_works is commented out for production code, this - # function will be never called. - def extract_table_name_from_insertion_query(sql) - $1 if sql =~ /INSERT INTO "(.*)" \(/ - end - - # Auxiliary function that extracts the sequence name. - # It's called by make_sure_pk_works. - # Ideally, if make_sure_pk_works is commented out for production code, this - # function will be never called. - def extract_sequence_name(seqStr) - $1 if seqStr =~ /\."(.*)"/ + # Assume that table name has one primary key column named id that is associated with a sequence, + # otherwise return + hdl = nil + sequence_name = extract_sequence_name(select_value("select \"default\" from #{MDB_SYS_SCHEMA}_columns where table_id in (select id from #{MDB_SYS_SCHEMA}_tables where name = '#{table_name}') and name='id';")) + + return if sequence_name.blank? + + max_id = select_value("select max(id) from #{table_name}").to_i + next_seq_val = select_value("select next value for #{sequence_name}").to_i + + if (max_id > next_seq_val) + hdl = execute("ALTER SEQUENCE #{sequence_name} RESTART WITH #{max_id+1}", name) + else + hdl = execute("ALTER SEQUENCE #{sequence_name} RESTART WITH #{next_seq_val+1}", name) end - + end + + # Auxiliary function that extracts the table name from an insertion query + # It's called by insert_sql in order to assist at make_sure_pk_works. + # Ideally, if make_sure_pk_works is commented out for production code, this + # function will be never called. + def extract_table_name_from_insertion_query(sql) + $1 if sql =~ /INSERT INTO "(.*)" \(/ + end + + # Auxiliary function that extracts the sequence name. + # It's called by make_sure_pk_works. + # Ideally, if make_sure_pk_works is commented out for production code, this + # function will be never called. + def extract_sequence_name(seqStr) + $1 if seqStr =~ /\."(.*)"/ + end + private - def connect - @connection.connect(user = @connection_options[2], passwd = @connection_options[3], lang = @connection_options[5], host = @connection_options[0], port = @connection_options[1], db_name = @connection_options[4], auth_type = "SHA1") if @connection - end + def connect + @connection.connect(user = @connection_options[2], passwd = @connection_options[3], lang = @connection_options[5], host = @connection_options[0], port = @connection_options[1], db_name = @connection_options[4], auth_type = "SHA1") if @connection + end end end end diff --git a/adapter/lib/active_record/connection_adapters/monetdb_adapter_config.rb b/adapter/lib/active_record/connection_adapters/monetdb_adapter_config.rb deleted file mode 100644 index 822b738..0000000 --- a/adapter/lib/active_record/connection_adapters/monetdb_adapter_config.rb +++ /dev/null @@ -1,13 +0,0 @@ -#MDB_SYS_SCHEMA = "" # initial value [before fix] - I.T. -MDB_SYS_SCHEMA = "sys." # explicitly specifying system schema - -#MDB_NON_SYSTEM_TABLES_ONLY = "" # initial value [before fix] - I.T. -MDB_NON_SYSTEM_TABLES_ONLY = "and system = false" # ignoring columns from system tables (in order not to mix in 'default_schema' and other fields from sys.users to OLDS users table) - -MDB_LOG_USER_QUERIES = false - -# NOTE: critical fix: lib/MonetDBData.rb, line 280: -# fields << f.gsub(/\\/, '').gsub(/^"/,'').gsub(/"$/,'').gsub(/\"/, '') -# should be -# fields << f.gsub(/\\n/, "\n").gsub(/\\/, '').gsub(/^"/,'').gsub(/"$/,'').gsub(/\"/, '') -# in order to treat line breaks correctly \ No newline at end of file diff --git a/lib/MonetDB.rb b/lib/MonetDB.rb index 2ce426d..d1936dc 100644 --- a/lib/MonetDB.rb +++ b/lib/MonetDB.rb @@ -254,8 +254,7 @@ def query(q="") if @connection != nil @data = MonetDBData.new(@connection) - (puts; puts q; puts) if MDB_LOG_USER_QUERIES # defined in adapter/lib/active_record/connection_adapter/monetdb_adapter_config.rb - @data.execute(q) + @data.execute(q) end return @data end diff --git a/lib/MonetDBConnection.rb b/lib/MonetDBConnection.rb index 5333def..e18df90 100644 --- a/lib/MonetDBConnection.rb +++ b/lib/MonetDBConnection.rb @@ -22,29 +22,28 @@ require 'socket' require 'time' require 'hasher' require 'MonetDBExceptions' -require 'iconv' # utf-8 support -require 'uri' # parse merovingian redirects +require 'uri' # parse merovingian redirects -Q_TABLE = "1" # SELECT operation -Q_UPDATE = "2" # INSERT/UPDATE operations -Q_CREATE = "3" # CREATE/DROP TABLE operations -Q_TRANSACTION = "4" # TRANSACTION -Q_PREPARE = "5" -Q_BLOCK = "6" # QBLOCK message +Q_TABLE = "1" # SELECT operation +Q_UPDATE = "2" # INSERT/UPDATE operations +Q_CREATE = "3" # CREATE/DROP TABLE operations +Q_TRANSACTION = "4" # TRANSACTION +Q_PREPARE = "5" +Q_BLOCK = "6" # QBLOCK message -MSG_REDIRECT = '^' # auth redirection through merovingian -MSG_QUERY = '&' -MSG_SCHEMA_HEADER = '%' -MSG_INFO = '!' # info response from mserver -MSG_TUPLE = '[' -MSG_PROMPT = "" +MSG_REDIRECT = '^' # auth redirection through merovingian +MSG_QUERY = '&' +MSG_SCHEMA_HEADER = '%' +MSG_INFO = '!' # info response from mserver +MSG_TUPLE = '[' +MSG_PROMPT = "" -REPLY_SIZE = '-1' +REPLY_SIZE = '-1' -MAX_AUTH_ITERATION = 10 # maximum number of atuh iterations (thorough merovingian) allowed - -MONET_ERROR = -1 +MAX_AUTH_ITERATION = 10 # maximum number of atuh iterations (thorough merovingian) allowed + +MONET_ERROR = -1 LANG_SQL = "sql" @@ -53,35 +52,35 @@ MAPIv8 = 8 MAPIv9 = 9 MONETDB_MEROVINGIAN = "merovingian" -MONETDB_MSERVER = "monetdb" +MONETDB_MSERVER = "monetdb" MEROVINGIAN_MAX_ITERATIONS = 10 class MonetDBConnection # enable debug output - @@DEBUG = false + @@DEBUG = false # hour in seconds, used for timezone calculation - @@HOUR = 3600 + @@HOUR = 3600 # maximum size (in bytes) for a monetdb message to be sent - @@MAX_MESSAGE_SIZE = 2048 - + @@MAX_MESSAGE_SIZE = 2048 + # endianness of a message sent to the server - @@CLIENT_ENDIANNESS = "BIG" - + @@CLIENT_ENDIANNESS = "BIG" + # MAPI protocols supported by the driver - @@SUPPORTED_PROTOCOLS = [ MAPIv8, MAPIv9 ] - + @@SUPPORTED_PROTOCOLS = [MAPIv8, MAPIv9] + attr_reader :socket, :auto_commit, :transactions, :lang - + # Instantiates a new MonetDBConnection object # * user: username (default is monetdb) # * passwd: password (default is monetdb) # * lang: language (default is sql) # * host: server hostanme or ip (default is localhost) # * port: server port (default is 50000) - + def initialize(user = "monetdb", passwd = "monetdb", lang = "sql", host="127.0.0.1", port = "50000") @user = user @passwd = passwd @@ -90,12 +89,12 @@ class MonetDBConnection @port = port @client_endianness = @@CLIENT_ENDIANNESS - + @auth_iteration = 0 @connection_established = false - + @transactions = MonetDBTransaction.new # handles a pool of transactions (generates and keeps track of savepoints) - + if @@DEBUG == true require 'logger' end @@ -105,27 +104,26 @@ class MonetDBConnection end end - + # Connect to the database, creates a new socket def connect(db_name = 'demo', auth_type = 'SHA1') @database = db_name @auth_type = auth_type - - @socket = TCPSocket.new(@host, @port.to_i) + + @socket = TCPSocket.new(@host, @port.to_i) if real_connect if @lang == LANG_SQL - set_timezone + set_timezone set_reply_size end true end false end - # perform a real connection; retrieve challenge, proxy through merovinginan, build challenge and set the timezone def real_connect - + server_challenge = retrieve_server_challenge() if server_challenge != nil salt = server_challenge.split(':')[0] @@ -139,7 +137,7 @@ class MonetDBConnection else raise MonetDBConnectionError, "Error: server returned an empty challenge string." end - + # The server supports only RIPMED168 or crypt as an authentication hash function, but the driver does not. if @supported_auth_types.length == 1 auth = @supported_auth_types[0] @@ -147,49 +145,49 @@ class MonetDBConnection raise MonetDBConnectionError, auth.upcase + " " + ": algorithm not supported by ruby-monetdb." end end - - + + # If the server protocol version is not 8: abort and notify the user. if @@SUPPORTED_PROTOCOLS.include?(@protocol) == false raise MonetDBProtocolError, "Protocol not supported. The current implementation of ruby-monetdb works with MAPI protocols #{@@SUPPORTED_PROTOCOLS} only." - + elsif mapi_proto_v8? reply = build_auth_string_v8(@auth_type, salt, @database) elsif mapi_proto_v9? reply = build_auth_string_v9(@auth_type, salt, @database) end - + if @socket != nil @connection_established = true send(reply) monetdb_auth = receive - + if monetdb_auth.length == 0 # auth succedeed true else if monetdb_auth[0].chr == MSG_REDIRECT - #redirection - + #redirection + redirects = [] # store a list of possible redirects - + monetdb_auth.split('\n').each do |m| # strip the trailing ^mapi: # if the redirect string start with something != "^mapi:" or is empty, the redirect is invalid and shall not be included. if m[0..5] == "^mapi:" redir = m[6..m.length] # url parse redir - redirects.push(redir) + redirects.push(redir) else $stderr.print "Warning: Invalid Redirect #{m}" - end + end end - - if redirects.size == 0 + + if redirects.size == 0 raise MonetDBConnectionError, "No valid redirect received" else - begin + begin uri = URI.split(redirects[0]) # Splits the string on following parts and returns array with result: # @@ -203,14 +201,14 @@ class MonetDBConnection # * Query # * Fragment server_name = uri[0] - host = uri[2] - port = uri[3] - database = uri[5].gsub(/^\//, '') if uri[5] != nil + host = uri[2] + port = uri[3] + database = uri[5].gsub(/^\//, '') if uri[5] != nil rescue URI::InvalidURIError raise MonetDBConnectionError, "Invalid redirect: #{redirects[0]}" end end - + if server_name == MONETDB_MEROVINGIAN if @auth_iteration <= MEROVINGIAN_MAX_ITERATIONS @auth_iteration += 1 @@ -226,8 +224,8 @@ class MonetDBConnection end # reinitialize a connection @host = host - @port = port - + @port = port + connect(database, @auth_type) else @connection_established = false @@ -239,33 +237,33 @@ class MonetDBConnection end end end + def savepoint @transactions.savepoint end # Formats a <i>command</i> string so that it can be parsed by the server def format_command(x) - return "X" + x + "\n" + return "X" + x + "\n" end - # send an 'export' command to the server def set_export(id, idx, offset) - send(format_command("export " + id.to_s + " " + idx.to_s + " " + offset.to_s )) + send(format_command("export " + id.to_s + " " + idx.to_s + " " + offset.to_s)) end - + # send a 'reply_size' command to the server def set_reply_size send(format_command(("reply_size " + REPLY_SIZE))) - + response = receive - + if response == MSG_PROMPT true elsif response[0] == MSG_INFO raise MonetDBCommandError, "Unable to set reply_size: #{response}" end - + end def set_output_seq @@ -274,7 +272,7 @@ class MonetDBConnection # Disconnect from server def disconnect() - if @connection_established + if @connection_established begin @socket.close rescue => e @@ -284,34 +282,34 @@ class MonetDBConnection raise MonetDBConnectionError, "No connection established." end end - + # send data to a monetdb5 server instance and returns server's response def send(data) encode_message(data).each do |m| @socket.write(m) end end - + # receive data from a monetdb5 server instance def receive is_final, chunk_size = recv_decode_hdr if chunk_size == 0 - return "" # needed on ruby-1.8.6 linux/64bit; recv(0) hangs on this configuration. + return "" # needed on ruby-1.8.6 linux/64bit; recv(0) hangs on this configuration. end - + data = @socket.recv(chunk_size) - - if is_final == false + + if is_final == false while is_final == false is_final, chunk_size = recv_decode_hdr - data += @socket.recv(chunk_size) + data += @socket.recv(chunk_size) end end - + return data end - + # Builds and authentication string given the parameters submitted by the user (MAPI protocol v8). # def build_auth_string_v8(auth_type, salt, db_name) @@ -320,20 +318,20 @@ class MonetDBConnection auth_type = auth_type.upcase digest = Hasher.new(auth_type, @passwd+salt) hashsum = digest.hashsum - elsif auth_type.downcase == "plain" or not @supported_auth_types.include?(auth_type.upcase) + elsif auth_type.downcase == "plain" or not @supported_auth_types.include?(auth_type.upcase) auth_type = 'plain' hashsum = @passwd + salt - + elsif auth_type.downcase == "crypt" - auth_type = @supported_auth_types[@supported_auth_types.index(auth_type)+1] + auth_type = @supported_auth_types[@supported_auth_types.index(auth_type)+1] $stderr.print "The selected hashing algorithm is not supported by the Ruby driver. #{auth_type} will be used instead." digest = Hasher.new(auth_type, @passwd+salt) hashsum = digest.hashsum else # The user selected an auth type not supported by the server. raise MonetDBConnectionError, "#{auth_type} not supported by the server. Please choose one from #{@supported_auth_types}" - - end + + end # Build the reply message with header reply = @client_endianness + ":" + @user + ":{" + auth_type + "}" + hashsum + ":" + @lang + ":" + db_name + ":" end @@ -346,57 +344,57 @@ class MonetDBConnection auth_type = auth_type.upcase # Hash the password pwhash = Hasher.new(@pwhash, @passwd) - + digest = Hasher.new(auth_type, pwhash.hashsum + salt) hashsum = digest.hashsum - + elsif auth_type.downcase == "plain" # or not @supported_auth_types.include?(auth_type.upcase) # Keep it for compatibility with merovingian auth_type = 'plain' hashsum = @passwd + salt elsif @supported_auth_types.include?(auth_type.upcase) if auth_type.upcase == "RIPEMD160" - auth_type = @supported_auth_types[@supported_auth_types.index(auth_type)+1] + auth_type = @supported_auth_types[@supported_auth_types.index(auth_type)+1] $stderr.print "The selected hashing algorithm is not supported by the Ruby driver. #{auth_type} will be used instead." end # Hash the password pwhash = Hasher.new(@pwhash, @passwd) - + digest = Hasher.new(auth_type, pwhash.hashsum + salt) - hashsum = digest.hashsum + hashsum = digest.hashsum else # The user selected an auth type not supported by the server. raise MonetDBConnectionError, "#{auth_type} not supported by the server. Please choose one from #{@supported_auth_types}" - end + end # Build the reply message with header reply = @client_endianness + ":" + @user + ":{" + auth_type + "}" + hashsum + ":" + @lang + ":" + db_name + ":" end # builds a message to be sent to the server - def encode_message(msg = "") + def encode_message(msg = "") message = Array.new data = "" - + hdr = 0 # package header pos = 0 is_final = false # last package in the stream - - while (! is_final) + + while (!is_final) data = msg[pos..pos+[@@MAX_MESSAGE_SIZE.to_i, (msg.length - pos).to_i].min] - pos += data.length - + pos += data.length + if (msg.length - pos) == 0 last_bit = 1 is_final = true else last_bit = 0 end - + hdr = [(data.length << 1) | last_bit].pack('v') - + message << hdr + data.to_s # Short Little Endian Encoding end - + message.freeze # freeze and return the encode message end @@ -404,13 +402,13 @@ class MonetDBConnection def retrieve_server_challenge() server_challenge = receive end - + # reads and decodes the header of a server message def recv_decode_hdr() if @socket != nil fb = @socket.recv(1) sb = @socket.recv(1) - + # Use execeptions handling to keep compatibility between different ruby # versions. # @@ -424,21 +422,21 @@ class MonetDBConnection fb = fb[0].to_i sb = sb[0].to_i end - + chunk_size = (sb << 7) | (fb >> 1) - + is_final = false - if ( (fb & 1) == 1 ) + if ((fb & 1) == 1) is_final = true - + end # return the size of the chunk (in bytes) - return is_final, chunk_size - else - raise MonetDBSocketError, "Error while receiving data\n" - end + return is_final, chunk_size + else + raise MonetDBSocketError, "Error while receiving data\n" + end end - + # Sets the time zone according to the Operating System settings def set_timezone() tz = Time.new @@ -456,32 +454,32 @@ class MonetDBConnection raise MonetDBQueryError, response end end - + # Turns auto commit on/off def set_auto_commit(flag=true) - if flag == false + if flag == false ac = " 0" - else + else ac = " 1" end send(format_command("auto_commit " + ac)) - + response = receive if response == MSG_PROMPT @auto_commit = flag elsif response[0].chr == MSG_INFO raise MonetDBCommandError, response - return + return end - + end - + # Check the auto commit status (on/off) def auto_commit? @auto_commit end - + # Check if monetdb is running behind the merovingian proxy and forward the connection in case def merovingian? if @server_name.downcase == MONETDB_MEROVINGIAN @@ -490,7 +488,7 @@ class MonetDBConnection false end end - + def mserver? if @server_name.downcase == MONETDB_MSERVER true @@ -498,7 +496,7 @@ class MonetDBConnection false end end - + # Check which protocol is spoken by the server def mapi_proto_v8? if @protocol == MAPIv8 @@ -507,44 +505,44 @@ class MonetDBConnection false end end - + def mapi_proto_v9? if @protocol == MAPIv9 true else false - end + end end end # handles transactions and savepoints. Can be used to simulate nested transactions. -class MonetDBTransaction +class MonetDBTransaction SAVEPOINT_STRING = "monetdbsp" - + def initialize @id = 0 @savepoint = "" end - + def savepoint @savepoint = SAVEPOINT_STRING + @id.to_s end - + def release prev_id end - + def save next_id end - + private def next_id @id += 1 end - + def prev_id @id -= 1 end - + end diff --git a/lib/MonetDBData.rb b/lib/MonetDBData.rb index 2ad7b4e..7dd8509 100644 --- a/lib/MonetDBData.rb +++ b/lib/MonetDBData.rb @@ -25,34 +25,34 @@ require 'MonetDBConnection' require 'logger' -class MonetDBData +class MonetDBData @@DEBUG = false attr_accessor :last_insert_id, :affected_rows - + def initialize(connection) @connection = connection @lang = @connection.lang # Structure containing the header+results set for a fired Q_TABLE query @header = [] - @query = {} - + @query = {} + @record_set = [] @index = 0 # Position of the last returned record - + @row_count = 0 @row_offset = 10 @row_index = Integer(REPLY_SIZE) end - + # Fire a query and return the server response def execute(q) - # fire a query and get ready to receive the data + # fire a query and get ready to receive the data @connection.send(format_query(q)) data = @connection.receive - + return if data == nil - + # temporarly store retrieved rows record_set = receive_record_set(data) @@ -61,7 +61,7 @@ class MonetDBData if @action == Q_TABLE @header = parse_header_table(@header) @header.freeze - + if @row_index.to_i < @row_count.to_i block_rows = "" while next_block @@ -75,18 +75,18 @@ class MonetDBData # ruby string management seems to not properly understand the MSG_PROMPT escape character. # In order to avoid data loss the @record_set array is built once that all tuples have been retrieved @record_set = record_set.split("\t]\n") - + if @record_set.length != @query['rows'].to_i raise MonetDBQueryError, "Warning: Query #{@query['id']} declared to result in #{@query['rows']} but #{@record_set.length} returned instead" end end - @record_set.freeze + @record_set.freeze end - + # Free memory used to store the record set def free() - @connection = nil - + @connection = nil + @header = [] @query = {} @@ -97,46 +97,46 @@ class MonetDBData @row_index = Integer(REPLY_SIZE) @row_count = 0 @row_offset = 10 - + end - + # Returns the record set entries hashed by column name orderd by column position def fetch_all_hash() - columns = {} - @header["columns_name"].each do |col_name| - columns[col_name] = fetch_column_name(col_name) - end + columns = {} + @header["columns_name"].each do |col_name| + columns[col_name] = fetch_column_name(col_name) + end - return columns - end + return columns + end def fetch_hash() - if @index >= @query['rows'].to_i - return false - else - columns = {} - @header["columns_name"].each do |col_name| - position = @header["columns_order"].fetch(col_name) - row = parse_tuple(@record_set[@index]) - columns[col_name] = row[position] - end - @index += 1 - return columns - end - end + if @index >= @query['rows'].to_i + return false + else + columns = {} + @header["columns_name"].each do |col_name| + position = @header["columns_order"].fetch(col_name) + row = parse_tuple(@record_set[@index]) + columns[col_name] = row[position] + end + @index += 1 + return columns + end + end # Returns the values for the column 'field' def fetch_column_name(field="") - position = @header["columns_order"].fetch(field) + position = @header["columns_order"].fetch(field) - col = Array.new - # Scan the record set by row - @record_set.each do |row| - col << parse_tuple(row)[position] - end + col = Array.new + # Scan the record set by row + @record_set.each do |row| + col << parse_tuple(row)[position] + end - return col - end + return col + end # returns result as an array of hashes def result_hashes @@ -154,58 +154,58 @@ class MonetDBData end def fetch() - @index - if @index > @query['rows'].to_i - false - else - parse_tuple(@record_set[@index]) - @index += 1 - end - end + @index + if @index > @query['rows'].to_i + false + else + parse_tuple(@record_set[@index]) + @index += 1 + end + end # Cursor method that retrieves all the records present in a table and stores them in a cache. def fetch_all() - if @query['type'] == Q_TABLE - rows = Array.new - @record_set.each do |row| - rows << parse_tuple(row) - end - @index = Integer(rows.length) - else - raise MonetDBDataError, "There is no record set currently available" + if @query['type'] == Q_TABLE + rows = Array.new + @record_set.each do |row| + rows << parse_tuple(row) end + @index = Integer(rows.length) + else + raise MonetDBDataError, "There is no record set currently available" + end + + return rows + end - return rows - end - # Returns the number of rows in the record set def num_rows() - return @query['rows'].to_i - end + return @query['rows'].to_i + end # Returns the number of fields in the record set def num_fields() - return @query['columns'].to_i - end + return @query['columns'].to_i + end # Returns the (ordered) name of the columns in the record set def name_fields() return @header['columns_name'] end - + # Returns the (ordered) name of the columns in the record set def type_fields return @header['columns_type'] end - + private - + # store block of data, parse it and store it. def receive_record_set(response) rows = "" lines = response.lines.to_a - response.each_line do |row| - if row[0].chr == MSG_QUERY + response.each_line do |row| + if row[0].chr == MSG_QUERY if row[1].chr == Q_TABLE @action = Q_TABLE @query = parse_header_query(row) @@ -214,7 +214,7 @@ class MonetDBData elsif row[1].chr == Q_BLOCK # strip the block header from data @action = Q_BLOCK - @block = parse_header_query(row) + @block = parse_header_query(row) elsif row[1].chr == Q_TRANSACTION @action = Q_TRANSACTION elsif row[1].chr == Q_CREATE @@ -240,10 +240,10 @@ class MonetDBData return rows end lines.shift - end + end rows # return an array of unparsed tuples end - + def next_block if REPLY_SIZE.to_i == -1 or @row_index == @row_count return false @@ -252,38 +252,37 @@ class MonetDBData # For larger values of the step performance drop; # @row_offset = [@row_offset, (@row_count - @row_index)].min - + # export offset amount - @connection.set_export(@query['id'], @row_index.to_s, @row_offset.to_s) - @row_index += @row_offset + @connection.set_export(@query['id'], @row_index.to_s, @row_offset.to_s) + @row_index += @row_offset @row_offset += 1 - end - return true + end + return true end - + # Formats a query <i>string</i> so that it can be parsed by the server def format_query(q) if @lang == LANG_SQL - return "s" + q + ";" + return "s" + q + ";" else raise LanguageNotSupported, @lang end end - + # parse one tuple as returned from the server def parse_tuple(tuple) fields = Array.new # remove trailing "[" - tuple = tuple.to_s.gsub(/^\[\s+/,'') - + tuple = tuple.to_s.gsub(/^\[\s+/, '') + tuple.split(/,\t/).each do |f| - #fields << f.gsub(/\\/, '').gsub(/^"/,'').gsub(/"$/,'').gsub(/\"/, '') - fields << f.gsub(/\\n/, "\n").gsub(/\\/, '').gsub(/^"/,'').gsub(/"$/,'').gsub(/\"/, '') + fields << f.gsub(/\\n/, "\n").gsub(/\\/, '').gsub(/^"/, '').gsub(/"$/, '').gsub(/\"/, '') end - + return fields.freeze end - + # Parses a query header and returns information about the query. def parse_header_query(row) type = row[1].chr @@ -293,54 +292,54 @@ class MonetDBData rows = row.split(' ')[2] columns = row.split(' ')[3] returned = row.split(' ')[4] - - header = { "id" => id, "type" => type, "rows" => rows, "columns" => columns, "returned" => returned } - elsif type == Q_BLOCK + + header = {"id" => id, "type" => type, "rows" => rows, "columns" => columns, "returned" => returned} + elsif type == Q_BLOCK # processing block header - + id = row.split(' ')[1] columns = row.split(' ')[2] remains = row.split(' ')[3] offset = row.split(' ')[4] - - header = { "id" => id, "type" => type, "remains" => remains, "columns" => columns, "offset" => offset } + + header = {"id" => id, "type" => type, "remains" => remains, "columns" => columns, "offset" => offset} else header = {"type" => type} end - + return header.freeze end - + # Parses a Q_TABLE header and returns information about the schema. def parse_header_table(header_t) if @query["type"] == Q_TABLE if header_t != nil name_t = header_t[0].split(' ')[1].gsub(/,$/, '') name_cols = Array.new - + header_t[1].split('%')[1].gsub(/'^\%'/, '').split('#')[0].split(' ').each do |col| name_cols << col.gsub(/,$/, '') end - - type_cols = { } + + type_cols = {} header_t[2].split('%')[1].gsub(/'^\%'/, '').split('#')[0].split(' ').each_with_index do |col, i| - if col.gsub(/,$/, '') != nil - type_cols[ name_cols[i] ] = col.gsub(/,$/, '') + if col.gsub(/,$/, '') != nil + type_cols[name_cols[i]] = col.gsub(/,$/, '') end end - - length_cols = { } + + length_cols = {} header_t[3].split('%')[1].gsub(/'^\%'/, '').split('#')[0].split(' ').each_with_index do |col, i| - length_cols[ name_cols[i] ] = col.gsub(/,$/, '') + length_cols[name_cols[i]] = col.gsub(/,$/, '') end - + columns_order = {} name_cols.each_with_index do |col, i| columns_order[col] = i end - - return {"table_name" => name_t, "columns_name" => name_cols, "columns_type" => type_cols, - "columns_length" => length_cols, "columns_order" => columns_order}.freeze + + return {"table_name" => name_t, "columns_name" => name_cols, "columns_type" => type_cols, + "columns_length" => length_cols, "columns_order" => columns_order}.freeze end end end @@ -351,44 +350,44 @@ class String def getInt self.to_i end - + def getFloat self.to_f end - + def getString - self.gsub(/^"/,'').gsub(/"$/,'') + self.gsub(/^"/, '').gsub(/"$/, '') end - + def getBlob # first strip trailing and leading " characters - self.gsub(/^"/,'').gsub(/"$/,'') - + self.gsub(/^"/, '').gsub(/"$/, '') + # convert from HEX to the origianl binary data. blob = "" self.scan(/../) { |tuple| blob += tuple.hex.chr } return blob end - + # ruby currently supports only time + date frommatted timestamps; # treat TIME and DATE as strings. def getTime # HH:MM:SS - self.gsub(/^"/,'').gsub(/"$/,'') + self.gsub(/^"/, '').gsub(/"$/, '') end - + def getDate - self.gsub(/^"/,'').gsub(/"$/,'') + self.gsub(/^"/, '').gsub(/"$/, '') end - + def getDateTime #YYYY-MM-DD HH:MM:SS date = self.split(' ')[0].split('-') time = self.split(' ')[1].split(':') - + Time.gm(date[0], date[1], date[2], time[0], time[1], time[2]) end - + def getChar # ruby < 1.9 does not have a Char datatype begin @@ -396,21 +395,21 @@ class String rescue c = self end - - return c + + return c end - + def getBool - if ['1', 'y', 't', 'true'].include?(self) - return true - elsif ['0','n','f', 'false'].include?(self) - return false - else - # unknown - return nil - end + if ['1', 'y', 't', 'true'].include?(self) + return true + elsif ['0', 'n', 'f', 'false'].include?(self) + return false + else + # unknown + return nil + end end - + def getNull if self.upcase == 'NONE' return nil diff --git a/ruby-monetdb-sql-0.1.gemspec b/ruby-monetdb-sql-0.1.gemspec index ed274f5..e773f07 100644 --- a/ruby-monetdb-sql-0.1.gemspec +++ b/ruby-monetdb-sql-0.1.gemspec @@ -1,8 +1,8 @@ Gem::Specification.new do |s| - s.required_ruby_version = '>= 1.8.0' + s.required_ruby_version = '>= 2.1.0' s.name = %q{ruby-monetdb-sql} - s.version = "0.1" + s.version = "0.2" s.date = %q{2009-04-27} s.authors = ["G Modena"] s.email = %q{gm@cwi.nl} -- GitLab