add 404 page and robots txt

 PATH
   remote: .
   specs:
-    kanjai (0.0.290)
+    kanjai (0.0.342)
       acts-as-taggable-on (~> 6.5)
       acts_as_list
       acts_as_tree
@@ -1194,7 +1194,7 @@ GEM
     aws-sigv2 (1.1.0)
     aws-sigv4 (1.4.0)
       aws-eventstream (~> 1, >= 1.0.2)
-    bcrypt (3.1.16)
+    bcrypt (3.1.18)
     builder (3.2.4)
     concurrent-ruby (1.1.9)
     crass (1.0.6)

@@ -45,7 +45,7 @@ module Kanjai
     private
 
     def permitted_params
-      params.permit(:domain => [:title, :default_lang_id, language_domains_attributes: [:id, :title, :page_lang_id, :_destroy], page_lang_ids: []])
+      params.permit(:domain => [:title, :default_lang_id, :robots, :page_404_id, language_domains_attributes: [:id, :title, :page_lang_id, :_destroy], page_lang_ids: []])
     end
 
   end

@@ -7,6 +7,9 @@ module Kanjai
 
 
     def show
+      render_status = @page_data.page == @page_data.page.domain.page_404 ? 401 : 200
+      domain = @page_data.page.domain
+
       if @page_data.page.block?
         render :template => 'kanjai/shared/401', status: 401, layout: false and return
       end
@@ -15,7 +18,11 @@ module Kanjai
         if @page_data.page.private_flag == true
           if defined?(warden).to_s == 'method'
             if warden.user.nil?
-              render :template => 'kanjai/shared/401', status: 401, layout: false and return
+              if domain.page_404.present?
+                redirect_to domain.page_404.menu_url(session[:scheme]) and return
+              else
+                render :template => 'kanjai/shared/401', status: 401, layout: false and return
+              end
             end
           end
         end
@@ -35,7 +42,7 @@ module Kanjai
             content_for_render = setTemplateContent(@page_data.page.domain, @page_data.page.page_template, layer, @page_data)
 
             respond_to do |format|
-              format.html{render :plain => content_for_render}
+              format.html{render :plain => content_for_render, status: render_status}
             end
 
             #render :text => content_for_render and return
@@ -44,18 +51,22 @@ module Kanjai
               layer = @page_data.page.page_template.get_html_content
               content_for_render = setTemplateContent(@page_data.page.domain, @page_data.page.page_template, layer, @page_data)
               respond_to do |format|
-                format.html{render :plain => content_for_render}
+                format.html{render :plain => content_for_render, status: render_status}
               end
             end
           end
         else
           page_content = get_html_by_json_client(@page_data, session)
-          render :html => page_content.html_safe and return
+          render :html => page_content.html_safe, status: render_status and return
         end
 
 
       else
-        render :template => 'kanjai/shared/404', status: 404, layout: false and return
+        if domain.page_404.present?
+          redirect_to domain.page_404.menu_url(session[:scheme]) and return
+        else
+          render :template => 'kanjai/shared/404', status: 404, layout: false and return
+        end
       end
     end
 

@@ -57,7 +57,7 @@ module Kanjai
             lang = domain.default_lang.try(:code)
           end
         else
-          render plain: "Domain #{domain_title} Not Exist" and return 
+          render plain: "Domain #{domain_title} Not Exist", status: 404 and return 
         end   
       else
         lang = domain.default_lang.try(:code)
@@ -91,10 +91,18 @@ module Kanjai
           if root_page
             redirect_to root_page.menu_url(session[:scheme])
           else
-            render plain: "1. Can not found page #{@url} for domain #{domain_title} for lang #{I18n.locale}", status: 404 and return
+            if domain.page_404.present?
+              redirect_to domain.page_404.menu_url(session[:scheme]) and return
+            else
+              render plain: "1. Can not found page #{@url} for domain #{domain_title} for lang #{I18n.locale}", status: 404 and return
+            end
           end
         else
-          render plain: "2. Can not found page #{@url} for domain #{domain_title} for lang #{I18n.locale}", status: 404 and return
+          if domain.page_404.present?
+            redirect_to domain.page_404.menu_url(session[:scheme]) and return
+          else          
+            render plain: "2. Can not found page #{@url} for domain #{domain_title} for lang #{I18n.locale}", status: 404 and return
+          end
         end
       end
       session[:url] = @url

 module Kanjai
   class Domain < ApplicationRecord
+    after_save :generate_robots_file
+
     validates :title, presence: true
 
     belongs_to :default_lang, class_name: 'Kanjai::PageLang', optional: true
@@ -8,5 +10,17 @@ module Kanjai
     has_and_belongs_to_many :page_langs
     has_many :language_domains, dependent: :destroy
     accepts_nested_attributes_for :language_domains, reject_if: :all_blank, allow_destroy: true
+
+    belongs_to :page_404, class_name: "Kanjai::Page", optional: true
+
+
+    private
+
+    def generate_robots_file
+      if saved_change_to_robots?
+        File.open("#{Rails.root}/public/robots.txt", 'w') { |file| file.write(robots) }
+      end
+    end
+
   end
 end

@@ -30,6 +30,18 @@
     </div>
   </div>
 
+  <div class="form-group">
+    <%= f.label :robots, Kanjai::Domain.human_attribute_name(:robots), class: 'col-form-label' %>
+    <%= f.text_area :robots, class: "form-control #{@domain.errors.include?(:robots) ? 'parsley-error' : ''} ", rows: 10 %>
+    <%= error_messages(@domain, :robots) %>
+  </div>  
+
+  <div class="form-group">
+    <%= f.label :page_404_id, Kanjai::Domain.human_attribute_name(:page_404_id), class: 'col-form-label' %>
+    <%= f.select :page_404_id, Kanjai::Page.pages_for_choose(@domain).collect{|p|  ["-" * p[:level]*2 +  p[:title], p[:id]] }, {:include_blank => true}, :class => "form-control" %>
+  </div>    
+
+
 
   <div class="card-footer mt-20">
     <div class="clearfix">

@@ -105,6 +105,8 @@ en:
       kanjai/domain:
         page_lang_ids: "Languages" 
         default_lang_id: "Default Language"
+        robots: "robots.txt"
+        page_404_id: "Page 404"
 
 
   actions:

+class AddRobotsToDomain < ActiveRecord::Migration[5.2]
+  def change
+    add_column :kanjai_domains, :robots, :text
+  end
+end

+class AddPage404IdToDomain < ActiveRecord::Migration[5.2]
+  def change
+    add_column :kanjai_domains, :page_404_id, :integer, index: true
+  end
+end

 module Kanjai
-  VERSION = "0.0.342"
+  VERSION = "0.0.343"
 end