diff --git a/Gemfile b/Gemfile
index 47251781ebcca4f2a51a6ab023e1b24005ad0a4d..cfcb968b1c9b7c34a1f224931644c280ebe7a814 100644
--- a/Gemfile
+++ b/Gemfile
@@ -14,6 +14,9 @@ gem 'bcrypt', '~> 3.1.7'
# memcached
gem 'dalli'
+# dalli multi thread gem
+gem 'connection_pool'
+
# web server
gem 'puma'
diff --git a/app/controllers/concerns/downloadable_controller.rb b/app/controllers/concerns/downloadable_controller.rb
new file mode 100644
index 0000000000000000000000000000000000000000..4ab7cfae8ef2728001f78bb147057f308f9c92c8
--- /dev/null
+++ b/app/controllers/concerns/downloadable_controller.rb
@@ -0,0 +1,16 @@
+module DownloadableController
+ extend ActiveSupport::Concern
+
+ # GET /learning_objects/1/download
+ def download
+ downloadable.download(current_user, request.remote_ip)
+ redirect_to downloadable.download_link
+ end
+
+ protected
+
+ def downloadable
+ raise NotImplementedError
+ end
+
+end
diff --git a/app/controllers/concerns/sociable_controller.rb b/app/controllers/concerns/sociable_controller.rb
index 0ffd5e8653ce565cc266abd775bf900504f88d0a..5f8f1e0921ceb1cee0258084012d4804648b69e9 100644
--- a/app/controllers/concerns/sociable_controller.rb
+++ b/app/controllers/concerns/sociable_controller.rb
@@ -33,12 +33,6 @@ module SociableController
end
end
- # GET /learning_objects/1/download
- def download
- sociable.download(current_user, request.remote_ip)
- redirect_to sociable.retrieve_link
- end
-
protected
def authorize_sociable!
diff --git a/app/controllers/v1/collections_controller.rb b/app/controllers/v1/collections_controller.rb
index b0dea3d5722d5ad0ecc3e0e839f3b85bbc7c9243..8faf84a832935c573de501cd7c2f964d33ec4276 100644
--- a/app/controllers/v1/collections_controller.rb
+++ b/app/controllers/v1/collections_controller.rb
@@ -1,5 +1,6 @@
class V1::CollectionsController < ApplicationController
include ::SociableController
+ include ::DownloadableController
include ::FollowableController
include ::TaggableController
include ::DeletedObjectsController
@@ -11,7 +12,7 @@ class V1::CollectionsController < ApplicationController
before_action :authenticate_user!, only: [:create, :update, :destroy]
before_action :set_collection, only: [:show, :update, :destroy, :add_object, :delete_object, :subjecting, :unsubjecting, :add_stages, :remove_stages]
before_action :set_new_collection, only: :index
- before_action :authorize!, except: [:create, :tagging, :untagging, :follow, :unfollow]
+ before_action :authorize!, except: [:create, :tagging, :untagging, :follow, :unfollow, :download]
# GET /v1/collections
# GET /v1/collections.json
@@ -83,6 +84,7 @@ class V1::CollectionsController < ApplicationController
def followable; set_collection; end
def taggable; set_collection; end
def sociable; set_collection; end
+ def downloadable; set_collection; end
def subjectable; set_collection; end
def stageable; set_collection; end
diff --git a/app/controllers/v1/learning_objects_controller.rb b/app/controllers/v1/learning_objects_controller.rb
index ec20bcd59767b035dc01fbe6aa39f0cb76a2fdbe..382495c988c643af92c2b0e08d226441e323fdff 100644
--- a/app/controllers/v1/learning_objects_controller.rb
+++ b/app/controllers/v1/learning_objects_controller.rb
@@ -2,6 +2,7 @@ require 'uri'
class V1::LearningObjectsController < ApplicationController
include ::SociableController
+ include ::DownloadableController
include ::TaggableController
include ::Paginator
include ::DeletedObjectsController
@@ -70,6 +71,7 @@ class V1::LearningObjectsController < ApplicationController
def deleted_resource; LearningObject; end
def highlights_resource; LearningObject; end
def sociable; set_learning_object; end
+ def downloadable; set_collection; end
def taggable; set_learning_object; end
def subjectable; set_learning_object; end
def stageable; set_learning_object; end
diff --git a/app/models/collection.rb b/app/models/collection.rb
index f3b7f2745949ffd7685e585600c73272172b3165..365a4666e9714c97412a654fc474a56a9eff84d7 100644
--- a/app/models/collection.rb
+++ b/app/models/collection.rb
@@ -25,6 +25,7 @@
class Collection < ApplicationRecord
include Reviewable
include Sociable
+ include Downloadable
include Followable
include Scoreable
include Thumbnailable
@@ -101,4 +102,8 @@ class Collection < ApplicationRecord
def user_category
owner.try('user_category')
end
+
+ def download_link
+ PackageService.link(self)
+ end
end
diff --git a/app/models/concerns/downloadable.rb b/app/models/concerns/downloadable.rb
new file mode 100644
index 0000000000000000000000000000000000000000..b813ac8371334bafc5a3394cc4cf020be5a17cda
--- /dev/null
+++ b/app/models/concerns/downloadable.rb
@@ -0,0 +1,21 @@
+module Downloadable
+ extend ActiveSupport::Concern
+
+ included do
+ has_many :downloads, as: :downloadable, dependent: :destroy
+ has_many :package_items, as: :packageable, dependent: :destroy
+ has_many :packages, through: :package_items, dependent: :destroy
+ end
+
+ def download(user, ip)
+ Download.where(user: user, ip: ip, downloadable: self).first_or_create
+ end
+
+ def downloaded?(user, ip)
+ !Download.where(user: user, ip: ip, downloadable: self).blank?
+ end
+
+ def download_link
+ raise NotImplementedError
+ end
+end
diff --git a/app/models/concerns/sociable.rb b/app/models/concerns/sociable.rb
index a01b35ddee65c0d12feb69b2fa4a34b8aa7a5e4a..2bc9fcb05f5896e3a1b7e2162afe4a1f52434f2f 100644
--- a/app/models/concerns/sociable.rb
+++ b/app/models/concerns/sociable.rb
@@ -3,7 +3,6 @@ module Sociable
included do
has_many :views, as: :viewable, dependent: :destroy
- has_many :downloads, as: :downloadable, dependent: :destroy
has_many :likes, as: :likeable, dependent: :destroy
has_many :shares, as: :shareable, dependent: :destroy
end
@@ -20,14 +19,6 @@ module Sociable
Like.where(user: user, likeable: self).destroy_all
end
- def download(user, ip)
- Download.where(user: user, ip: ip, downloadable: self).first_or_create
- end
-
- def downloaded?(user, ip)
- !Download.where(user: user, ip: ip, downloadable: self).blank?
- end
-
def share(user)
Share.create(user: user, shareable: self)
end
diff --git a/app/models/learning_object.rb b/app/models/learning_object.rb
index 6f8678bfd02b61282f35c12517855a286c39bc22..857bd9c086505a0c74d7c6489063a54fa6a3c22d 100644
--- a/app/models/learning_object.rb
+++ b/app/models/learning_object.rb
@@ -34,6 +34,7 @@ class LearningObject < ApplicationRecord
include Metadatable
include Reviewable
include Sociable
+ include Downloadable
include Stateful
include Scoreable
include Thumbnailable
@@ -122,6 +123,10 @@ class LearningObject < ApplicationRecord
default_attachment.retrieve_cache_link
end
+ def download_link
+ retrieve_link
+ end
+
## score methods
def normalized_collected
max = CollectionItem.where(collectionable_type: 'LearningObject').group(:collectionable_id).order('count_all DESC').count
diff --git a/app/models/package.rb b/app/models/package.rb
new file mode 100644
index 0000000000000000000000000000000000000000..2bb87a743c1f83217793ca928d09e79029a45e8e
--- /dev/null
+++ b/app/models/package.rb
@@ -0,0 +1,13 @@
+class Package < ApplicationRecord
+ has_many :package_items, dependent: :destroy
+
+ has_attached_file :file, path: ":existent_file_or_default"
+
+ validates_presence_of :file
+
+ def add_items(items)
+ items.each do |item|
+ package_item = PackageItem.where(package: self, packageable_id: item[:id], packageable_type: item[:type]).first_or_create
+ end
+ end
+end
diff --git a/app/models/package_item.rb b/app/models/package_item.rb
new file mode 100644
index 0000000000000000000000000000000000000000..e9f88646320bca39254759ef69fa81bc31f7a9a1
--- /dev/null
+++ b/app/models/package_item.rb
@@ -0,0 +1,6 @@
+class PackageItem < ApplicationRecord
+ belongs_to :package
+ belongs_to :packageable, polymorphic: true
+
+ validates :package, :packageable, presence: true
+end
diff --git a/app/services/package_service/generator.rb b/app/services/package_service/generator.rb
index ded3003f5f14968f4454c4bdb5e8ec840790f61c..98f7698580037a14dca73e92f331c0570888073c 100644
--- a/app/services/package_service/generator.rb
+++ b/app/services/package_service/generator.rb
@@ -1,15 +1,22 @@
module PackageService
class Generator
+
+ attr_reader :objects
+
def initialize(objects)
@objects = objects.is_a?(Array) ? objects : [objects]
@cache_key = nil
+ @filename = nil
raise 'Invalid objects for PackageService' unless valid?
end
def generate
- Rails.cache.write(PackageService.job_key(cache_key), 'wait')
+ # Rails.cache.write(PackageService.job_key(cache_key), 'wait')
PackageWorker.perform_async(objects_map, filename, cache_key)
+
+ # p "-------- Generator - cache data"
+ # p Rails.cache.instance_variable_get("@data").keys
end
def cache_key
@@ -24,6 +31,14 @@ module PackageService
@cache_key = "#{key_name}[#{cache_list.join(',')}]"
end
+ def filename
+ while @filename.nil? do
+ name = "#{SecureRandom.hex(12)}.zip"
+ @filename = name unless File.exist?(PackageService.file_root(name))
+ end
+ @filename
+ end
+
private
def valid?
@@ -36,13 +51,6 @@ module PackageService
true
end
- def filename
- loop do
- name = "#{SecureRandom.hex(12)}.zip"
- return name unless File.exist?(PackageService.file_root(name))
- end
- end
-
def objects_map
@objects.map { |o| { class: o.class, id: o.id } }
end
diff --git a/app/services/package_service/link.rb b/app/services/package_service/link.rb
index 897fdc5f8ddcda37b9186ef3c683bbd1d6da25e8..d7ab3691733b21c9594b8820315f5bf48f899138 100644
--- a/app/services/package_service/link.rb
+++ b/app/services/package_service/link.rb
@@ -1,43 +1,96 @@
module PackageService
class Link
- def initialize(object)
- @generator = Generator.new(object)
+ def initialize(objects)
+ @generator = Generator.new(objects)
@cache_key = @generator.cache_key
end
def link
- link = retrieve_link
-
+ link = get_package_link
+ p "---------- Link - before generate"
+ p link
if link.nil?
+ p "------ generating"
@generator.generate
link = retrieve_link
end
-
- "#{PackageService.dirname}/#{link}" unless link.nil?
+ p "--------------- Link - link"
+ p link
+ # "#{PackageService.dirname}/#{link}" unless link.nil?
+ link
end
private
+ def get_package_link
+ package = PackageItem.select("package_id").where(:packageable => @generator.objects).group("package_id")
+ return nil if package.blank?
+ p "------- package"
+ p package
+ Package.find(package.first).file.path
+ end
+
def retrieve_link
if wait_job
- filename = Rails.cache.fetch(@cache_key)
- return filename if !filename.nil? && File.exist?(PackageService.file_root(filename))
+ p "------ entrou no if"
+ # filename = Rails.cache.fetch(@cache_key)
+ filename = @generator.filename
+ p "--------- filename"
+ p filename
+ p PackageService.file_root(filename)
+ p File.exist?(PackageService.file_root(filename))
- Rails.cache.delete(@cache_key) if Rails.cache.exist?(@cache_key)
+ filepath = 'public/'+PackageService.dirname+'/'+filename
+ p "---------- filepath"
+ p filepath
+ package = Package.create({
+ file_path: filepath,
+ file_file_name: filename,
+ file_content_type: "application/zip",
+ file_file_size: File.size(filepath)
+ })
+ package.add_items(@generator.objects)
+ return package.file
+ # return filename if !filename.nil? && File.exist?(PackageService.file_root(filename))
+ # p "-------- wait_job - cache data"
+ # p Rails.cache.instance_variable_get("@data").keys
+ # Rails.cache.delete(@cache_key) if Rails.cache.exist?(@cache_key)
end
nil
end
def wait_job
- job_key = PackageService.job_key(@cache_key)
- unless Rails.cache.fetch(job_key).nil?
+ # job_key = PackageService.job_key(@cache_key)
+ # p "-------- job_key"
+ # p job_key
+
+ # p "-------- Link - cache data"
+ # p Rails.cache.instance_variable_get("@data").keys
+ # unless Rails.cache.fetch(job_key).nil?
+ # Timeout.timeout(60) do
+ # begin
+ # p "-------- job_key deleted?"
+ # p Rails.cache.fetch(job_key).nil?
+ # p "-------- cache_key found?"
+ # p !Rails.cache.fetch(@cache_key).nil?
+ # sleep(1.0)
+ # end until Rails.cache.fetch(job_key).nil? && !Rails.cache.fetch(@cache_key).nil?
+ # end
+ # end
+ #FIXME: Nome do arquivo é aleatório então vai gerar um novo a cada requisição
+ #TODO: Criar um modelo do Package com relação entre lo e col p/ saber se já teve o package criado e qual o caminho
+ unless File.exist?(PackageService.file_root(@generator.filename))
Timeout.timeout(60) do
- sleep(1.0) until Rails.cache.fetch(job_key).nil? && !Rails.cache.fetch(@cache_key).nil?
+ begin
+ p "."
+ sleep(1.0)
+ end until File.exist?(PackageService.file_root(@generator.filename))
end
end
true
rescue Timeout::Error
+ p "------------ TIMEOUT"
false
end
end
diff --git a/app/workers/package_worker.rb b/app/workers/package_worker.rb
index cfaa9712ba25cdc8ffdc46393ac79453bc5e6fe1..8e2a203751e0d4709fd70de187431d8cd16052a4 100644
--- a/app/workers/package_worker.rb
+++ b/app/workers/package_worker.rb
@@ -1,3 +1,5 @@
+require_dependency 'dspace'
+require 'rails'
class PackageWorker
include Sidekiq::Worker
require 'zip'
@@ -5,30 +7,35 @@ class PackageWorker
sidekiq_options queue: :package_cache
def perform(objects_ids = nil, filename = nil, cache_key = nil)
+ p "--------- comecou worker"
+ # p "--------- cache_key"
+ # p cache_key
return false if objects_ids.blank? || filename.blank? || cache_key.blank?
@cache_key = cache_key
- return true if file_exist?(PackageService.file_root(filename))
+ return true if File.exist?(PackageService.file_root(filename))
- Rails.cache.write(PackageService.job_key(cache_key), 'wait')
+ # Rails.cache.write(PackageService.job_key(cache_key), 'wait')
files = open_files(objects_ids)
create_package(filename, files)
ensure
close_files(files) unless files.nil?
- job_key = PackageService.job_key(cache_key)
- Rails.cache.delete(job_key) if Rails.cache.exist?(job_key)
+ # p "-------- PackageWorker - cache data"
+ # p Rails.cache.instance_variable_get("@data").keys
+ # job_key = PackageService.job_key(cache_key)
+ # Rails.cache.delete(job_key) if Rails.cache.exist?(job_key)
end
private
- def file_exist?(path)
- if File.exist?(path)
- cache_fetch(path.split('/').last)
- return true
- end
- false
- end
+ # def file_exist?(path)
+ # if File.exist?(path)
+ # cache_fetch(path.split('/').last)
+ # return true
+ # end
+ # false
+ # end
def open_files(objects_ids)
objects = objects_ids.map { |o| o['class'].constantize.find(o['id']) }
@@ -50,18 +57,26 @@ class PackageWorker
object.attachments.where(bundle_name: 'ORIGINAL').each do |a|
link = a.retrieve_cache_link
link = a.id_dspace.to_s if link =~ %r{^https?:\/\/.*}i
+ p "------- object_files"
+ p link
+ #FIXME: NameError: uninitialized constant Dspace::Client
file = open_file(link)
+ p file
files << file unless file.nil?
end
files
end
def create_package(filename, files)
+ p "--------- create_package"
+ p files
FileUtils.mkdir_p(PackageService.file_root)
Zip::File.open(PackageService.file_root(filename), Zip::File::CREATE) do |zipfile|
files.each { |file| zipfile.add(File.basename(file.path), file.path) }
end
- cache_fetch(filename)
+ p "--------- zip filename"
+ p filename
+ # cache_fetch(filename)
rescue => e
file = PackageService.file_root(filename)
FileUtils.rm(file) if File.exist?(file)
@@ -69,8 +84,12 @@ class PackageWorker
end
def open_file(file_path)
- path = Rails.root.join('public', file_path)
+ # path = Rails.root.join("public", file_path)
+ path = Rails.root.join("public" + file_path)
+ p "-------- path"
+ p path
return File.open(path) if File.exist? path
+ p "-----------não achou"
open_dspace_file(file_path)
end
@@ -92,12 +111,14 @@ class PackageWorker
end
end
- def cache_fetch(value)
- filename = Rails.cache.fetch(@cache_key)
- file = PackageService.file_root(filename) unless filename.nil?
- FileUtils.rm(file) if !file.nil? && File.exist?(file)
-
- Rails.cache.delete(@cache_key)
- Rails.cache.write(@cache_key, value)
- end
+ # def cache_fetch(value)
+ # filename = Rails.cache.fetch(@cache_key)
+ # file = PackageService.file_root(filename) unless filename.nil?
+ # FileUtils.rm(file) if !file.nil? && File.exist?(file)
+ #
+ # Rails.cache.delete(@cache_key)
+ # #FIXME: Só funciona na segunda tentativa e tem q arrumar o link
+ # p "------- escreveu"
+ # Rails.cache.write(@cache_key, value)
+ # end
end
diff --git a/config/environments/development.rb b/config/environments/development.rb
index 9724cd4866a8248ceb1b79d1ec045fd170079787..6ae5d8734cdcacfaba82553849f2a1aeabc850a0 100644
--- a/config/environments/development.rb
+++ b/config/environments/development.rb
@@ -28,7 +28,8 @@ Rails.application.configure do
if Rails.root.join('tmp/caching-dev.txt').exist?
config.action_controller.perform_caching = true
- config.cache_store = :memory_store
+ config.cache_store = :dalli_store, nil, { :namespace => 'portalmec', :expires_in => 1.day, :compress => true, :pool_size => 5 }
+
config.public_file_server.headers = {
'Cache-Control' => 'public, max-age=172800'
}
diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb
new file mode 100644
index 0000000000000000000000000000000000000000..8c2695f7f4fa6eedc0707bac98201c5d69328613
--- /dev/null
+++ b/config/initializers/paperclip.rb
@@ -0,0 +1,4 @@
+Paperclip.interpolates :existent_file_or_default do |attachment, style|
+ attachment.instance.file_path ||
+ attachment.interpolator.interpolate(":class/:attachment/:id_partition/:style/:filename", attachment, style)
+end
diff --git a/config/routes.rb b/config/routes.rb
index 0015aa1816392eba7aef377a4d4312e192e6b313..37fa9156f50a7b0f549652d6ebe632ba73496672 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -40,6 +40,11 @@ Rails.application.routes.draw do
member do
post 'like', as: :like, action: :like
delete 'like', as: :unlike, action: :unlike
+ end
+ end
+
+ concern :downloadable do
+ member do
get 'download', as: :download, action: :download
end
end
@@ -110,14 +115,14 @@ Rails.application.routes.draw do
end
end
- resources :collections, concerns: [:followable, :sociable, :reviewable, :taggable, :versionable, :deletable, :highlights, :subjectable, :stageable] do
+ resources :collections, concerns: [:followable, :sociable, :downloadable, :reviewable, :taggable, :versionable, :deletable, :highlights, :subjectable, :stageable] do
member do
post :items, to: 'collections#add_object'
delete :items, to: 'collections#delete_object'
end
end
- resources :learning_objects, concerns: [:sociable, :reviewable, :taggable, :versionable, :deletable, :highlights, :subjectable, :stageable] do
+ resources :learning_objects, concerns: [:sociable, :downloadable, :reviewable, :taggable, :versionable, :deletable, :highlights, :subjectable, :stageable] do
member do
resource :chunk, module: 'learning_objects', only: [:create, :show]
resource :upload, module: 'learning_objects', only: :create
diff --git a/db/migrate/20170223132327_create_packages.rb b/db/migrate/20170223132327_create_packages.rb
new file mode 100644
index 0000000000000000000000000000000000000000..c2e22f4584b5e4b9fa51e545a59f11e7d4ef8b5f
--- /dev/null
+++ b/db/migrate/20170223132327_create_packages.rb
@@ -0,0 +1,9 @@
+class CreatePackages < ActiveRecord::Migration[5.0]
+ def change
+ create_table :packages do |t|
+ t.references :package_items, index: true
+
+ t.timestamps
+ end
+ end
+end
diff --git a/db/migrate/20170223132838_create_package_items.rb b/db/migrate/20170223132838_create_package_items.rb
new file mode 100644
index 0000000000000000000000000000000000000000..14352c19c28f655183b8f548227c2eb391af77d2
--- /dev/null
+++ b/db/migrate/20170223132838_create_package_items.rb
@@ -0,0 +1,10 @@
+class CreatePackageItems < ActiveRecord::Migration[5.0]
+ def change
+ create_table :package_items do |t|
+ t.references :package, foreign_key: true
+ t.references :packageable, polymorphic: true
+
+ t.timestamps
+ end
+ end
+end
diff --git a/db/migrate/20170223140805_add_attachment_file_to_packages.rb b/db/migrate/20170223140805_add_attachment_file_to_packages.rb
new file mode 100644
index 0000000000000000000000000000000000000000..1bcb4257d14971d9fbdf5d786653179064457ee7
--- /dev/null
+++ b/db/migrate/20170223140805_add_attachment_file_to_packages.rb
@@ -0,0 +1,11 @@
+class AddAttachmentFileToPackages < ActiveRecord::Migration
+ def self.up
+ change_table :packages do |t|
+ t.attachment :file
+ end
+ end
+
+ def self.down
+ remove_attachment :packages, :file
+ end
+end
diff --git a/db/migrate/20170224145548_add_filepath_to_package.rb b/db/migrate/20170224145548_add_filepath_to_package.rb
new file mode 100644
index 0000000000000000000000000000000000000000..4ebbf5091493393108594f9a9514897e76480f0f
--- /dev/null
+++ b/db/migrate/20170224145548_add_filepath_to_package.rb
@@ -0,0 +1,5 @@
+class AddFilepathToPackage < ActiveRecord::Migration[5.0]
+ def change
+ add_column :packages, :file_path, :string
+ end
+end