def pbAddScriptTexts(items, script)
  script.force_encoding(Encoding::UTF_8)
  script.scan(/(?:_I)\s*\(\s*\"((?:[^\\\"]*\\\"?)*[^\"]*)\"/) do |s|
    string = s[0]
    string.gsub!(/\\\"/, "\"")
    string.gsub!(/\\\\/, "\\")
    items.push(string)
  end
end

def pbAddRgssScriptTexts(items, script)
  script.force_encoding(Encoding::UTF_8)
  script.scan(/(?:_INTL|_ISPRINTF)\s*\(\s*\"((?:[^\\\"]*\\\"?)*[^\"]*)\"/) do |s|
    string = s[0]
    string.gsub!(/\\r/, "\r")
    string.gsub!(/\\n/, "\n")
    string.gsub!(/\\1/, "\1")
    string.gsub!(/\\\"/, "\"")
    string.gsub!(/\\\\/, "\\")
    items.push(string)
  end
end

def pbSetTextMessages
  Graphics.update
  begin
    t = System.uptime
    texts = []
    # Get script texts from Scripts.rxdata
    $RGSS_SCRIPTS.each do |script|
      if System.uptime - t >= 5
        t += 5
        Graphics.update
      end
      scr = Zlib::Inflate.inflate(script[2])
      pbAddRgssScriptTexts(texts, scr)
    end
    # If Scripts.rxdata only has 1 section, scripts have been extracted. Get
    # script texts from .rb files in Data/Scripts
    if $RGSS_SCRIPTS.length == 1
      Dir.all("Data/Scripts").each do |script_file|
        if System.uptime - t >= 5
          t += 5
          Graphics.update
        end
        File.open(script_file, "rb") do |f|
          pbAddRgssScriptTexts(texts, f.read)
        end
      end
    end
    # Must add messages because this code is used by both game system and Editor
    MessageTypes.addMessagesAsHash(MessageTypes::ScriptTexts, texts)
    commonevents = load_data("Data/CommonEvents.rxdata")
    items = []
    choices = []
    for event in commonevents.compact
      if Time.now.to_i - t >= 5
        t = Time.now.to_i
        Graphics.update
      end
      begin
        neednewline = false
        lastitem = ""
        for j in 0...event.list.size
          list = event.list[j]
          if neednewline && list.code != 401
            if lastitem != ""
              lastitem.gsub!(/([^\.\!\?])\s\s+/) { |m| $1 + " " }
              items.push(lastitem)
              lastitem = ""
            end
            neednewline = false
          end
          if list.code == 101
            lastitem += "#{list.parameters[0]}"
            neednewline = true
          elsif list.code == 102
            for k in 0...list.parameters[0].length
              choices.push(list.parameters[0][k])
            end
            neednewline = false
          elsif list.code == 401
            lastitem += " " if lastitem != ""
            lastitem += "#{list.parameters[0]}"
            neednewline = true
          elsif list.code == 355 || list.code == 655
            pbAddScriptTexts(items, list.parameters[0])
          elsif list.code == 111 && list.parameters[0] == 12
            pbAddScriptTexts(items, list.parameters[1])
          elsif list.code == 209
            route = list.parameters[1]
            for k in 0...route.list.size
              if route.list[k].code == 45
                pbAddScriptTexts(items, route.list[k].parameters[0])
              end
            end
          end
        end
        if neednewline
          if lastitem != ""
            items.push(lastitem)
            lastitem = ""
          end
        end
      end
    end
    if Time.now.to_i - t >= 5
      t = Time.now.to_i
      Graphics.update
    end
    items |= []
    choices |= []
    items.concat(choices)
    MessageTypes.setMapMessagesAsHash(0, items)
    mapinfos = pbLoadMapInfos
    mapnames = []
    for id in mapinfos.keys
      mapnames[id] = mapinfos[id].name
    end
    MessageTypes.setMessages(MessageTypes::MapNames, mapnames)
    for id in mapinfos.keys
      if Time.now.to_i - t >= 5
        t = Time.now.to_i
        Graphics.update
      end
      filename = sprintf("Data/Map%03d.rxdata", id)
      next if !pbRgssExists?(filename)
      map = load_data(filename)
      items = []
      choices = []
      for event in map.events.values
        if Time.now.to_i - t >= 5
          t = Time.now.to_i
          Graphics.update
        end
        begin
          for i in 0...event.pages.size
            neednewline = false
            lastitem = ""
            for j in 0...event.pages[i].list.size
              list = event.pages[i].list[j]
              if neednewline && list.code != 401
                if lastitem != ""
                  lastitem.gsub!(/([^\.\!\?])\s\s+/) { |m| $1 + " " }
                  items.push(lastitem)
                  lastitem = ""
                end
                neednewline = false
              end
              if list.code == 101
                lastitem += "#{list.parameters[0]}"
                neednewline = true
              elsif list.code == 102
                for k in 0...list.parameters[0].length
                  choices.push(list.parameters[0][k])
                end
                neednewline = false
              elsif list.code == 401
                lastitem += " " if lastitem != ""
                lastitem += "#{list.parameters[0]}"
                neednewline = true
              elsif list.code == 355 || list.code == 655
                pbAddScriptTexts(items, list.parameters[0])
              elsif list.code == 111 && list.parameters[0] == 12
                pbAddScriptTexts(items, list.parameters[1])
              elsif list.code == 209
                route = list.parameters[1]
                for k in 0...route.list.size
                  if route.list[k].code == 45
                    pbAddScriptTexts(items, route.list[k].parameters[0])
                  end
                end
              end
            end
            if neednewline
              if lastitem != ""
                items.push(lastitem)
                lastitem = ""
              end
            end
          end
        end
      end
      if Time.now.to_i - t >= 5
        t = Time.now.to_i
        Graphics.update
      end
      items |= []
      choices |= []
      items.concat(choices)
      MessageTypes.setMapMessagesAsHash(id, items)
      if Time.now.to_i - t >= 5
        t = Time.now.to_i
        Graphics.update
      end
    end
  rescue Hangup
  end
  Graphics.update
end

def pbEachIntlSection(file)
  lineno = 1
  re = /^\s*\[\s*([^\]]+)\s*\]\s*$/
  havesection = false
  sectionname = nil
  lastsection = []
  file.each_line { |line|
    if lineno == 1 && line[0].ord == 0xEF && line[1].ord == 0xBB && line[2].ord == 0xBF
      line = line[3, line.length - 3]
    end
    if !line[/^\#/] && !line[/^\s*$/]
      if line[re]
        if havesection
          yield lastsection, sectionname
        end
        lastsection.clear
        sectionname = $~[1]
        havesection = true
      else
        if sectionname == nil
          raise _INTL("Expected a section at the beginning of the file (line {1})", lineno)
        end
        lastsection.push(line.gsub(/\s+$/, ""))
      end
    end
    lineno += 1
    if lineno % 500 == 0
      Graphics.update
    end
  }
  if havesection
    yield lastsection, sectionname
  end
end

def pbGetText(infile)
  begin
    file = File.open(infile, "rb")
  rescue
    raise _INTL("Can't find {1}", infile)
  end
  intldat = []
  begin
    pbEachIntlSection(file) { |section, name|
      next if section.length == 0
      if !name[/^([Mm][Aa][Pp])?(\d+)$/]
        raise _INTL("Invalid section name {1}", name)
      end
      ismap = $~[1] && $~[1] != ""
      id = $~[2].to_i
      itemlength = 0
      if section[0][/^\d+$/]
        intlhash = []
        itemlength = 3
        if ismap
          raise _INTL("Section {1} can't be an ordered list (section was recognized as an ordered list because its first line is a number)", name)
        end
        if section.length % 3 != 0
          raise _INTL("Section {1}'s line count is not divisible by 3 (section was recognized as an ordered list because its first line is a number)", name)
        end
      else
        intlhash = OrderedHash.new
        itemlength = 2
        if section.length % 2 != 0
          raise _INTL("Section {1} has an odd number of entries (section was recognized as a hash because its first line is not a number)", name)
        end
      end
      i = 0
      loop do break unless i < section.length
        if itemlength == 3
        if !section[i][/^\d+$/]
          raise _INTL("Expected a number in section {1}, got {2} instead", name, section[i])
        end
        key = section[i].to_i
        i += 1
      else
        key = MessageTypes.denormalizeValue(section[i])
      end
        intlhash[key] = MessageTypes.denormalizeValue(section[i + 1])
        i += 2       end
      if ismap
        intldat[0] = [] if !intldat[0]
        intldat[0][id] = intlhash
      else
        intldat[id] = intlhash
      end
    }
  ensure
    file.close
  end
  return intldat
end

def pbCompileText
  outfile = File.open("intl.dat", "wb")
  begin
    intldat = pbGetText("intl.txt")
    Marshal.dump(intldat, outfile)
  rescue
    raise
  ensure
    outfile.close
  end
end

class OrderedHash < Hash
  def initialize
    @keys = []
    super
  end

  def keys
    return @keys.clone
  end

  def inspect
    str = "{"
    for i in 0...@keys.length
      str += ", " if i > 0
      str += @keys[i].inspect + "=>" + self[@keys[i]].inspect
    end
    str += "}"
    return str
  end

  alias :to_s :inspect

  def []=(key, value)
    oldvalue = self[key]
    if !oldvalue && value
      @keys.push(key)
    elsif !value
      @keys |= []
      @keys -= [key]
    end
    return super(key, value)
  end

  def self._load(string)
    ret = self.new
    keysvalues = Marshal.load(string)
    keys = keysvalues[0]
    values = keysvalues[1]
    for i in 0...keys.length
      ret[keys[i]] = values[i]
    end
    return ret
  end

  def _dump(_depth = 100)
    values = []
    for key in @keys
      values.push(self[key])
    end
    return Marshal.dump([@keys, values])
  end
end

class Messages
  def initialize(filename = nil, delayLoad = false)
    @messages = nil
    @filename = filename
    if @filename && !delayLoad
      loadMessageFile(@filename)
    end
  end

  def delayedLoad
    if @filename && !@messages
      loadMessageFile(@filename)
      @filename = nil
    end
  end

  def self.stringToKey(str)
    if str && str[/[\r\n\t\1]|^\s+|\s+$|\s{2,}/]
      key = str.clone
      key.gsub!(/^\s+/, "")
      key.gsub!(/\s+$/, "")
      key.gsub!(/\s{2,}/, " ")
      return key
    end
    return str
  end

  def self.normalizeValue(value)
    if value[/[\r\n\t\x01]|^[\[\]]/]
      ret = value.clone
      ret.gsub!(/\r/, "<<r>>")
      ret.gsub!(/\n/, "<<n>>")
      ret.gsub!(/\t/, "<<t>>")
      ret.gsub!(/\[/, "<<[>>")
      ret.gsub!(/\]/, "<<]>>")
      ret.gsub!(/\x01/, "<<1>>")
      return ret
    end
    return value
  end

  def self.denormalizeValue(value)
    if value[/<<[rnt1\[\]]>>/]
      ret = value.clone
      ret.gsub!(/<<1>>/, "\1")
      ret.gsub!(/<<r>>/, "\r")
      ret.gsub!(/<<n>>/, "\n")
      ret.gsub!(/<<\[>>/, "[")
      ret.gsub!(/<<\]>>/, "]")
      ret.gsub!(/<<t>>/, "\t")
      return ret
    end
    return value
  end

  def self.writeObject(f, msgs, secname, origMessages = nil)
    return if !msgs
    if msgs.is_a?(Array)
      f.write("[#{secname}]\r\n")
      for j in 0...msgs.length
        next if nil_or_empty?(msgs[j])
        value = Messages.normalizeValue(msgs[j])
        origValue = ""
        if origMessages
          origValue = Messages.normalizeValue(origMessages.get(secname, j))
        else
          origValue = Messages.normalizeValue(MessageTypes.get(secname, j))
        end
        f.write("#{j}\r\n")
        f.write(origValue + "\r\n")
        f.write(value + "\r\n")
      end
    elsif msgs.is_a?(OrderedHash)
      f.write("[#{secname}]\r\n")
      keys = msgs.keys
      for key in keys
        next if nil_or_empty?(msgs[key])
        value = Messages.normalizeValue(msgs[key])
        valkey = Messages.normalizeValue(key)
        # key is already serialized
        f.write(valkey + "\r\n")
        f.write(value + "\r\n")
      end
    end
  end

  def messages
    return @messages || []
  end

  def extract(outfile)
    #    return if !@messages
    origMessages = Messages.new("Data/messages.dat")
    File.open(outfile, "wb") { |f|
      f.write(0xef.chr)
      f.write(0xbb.chr)
      f.write(0xbf.chr)
      f.write("# To localize this text for a particular language, please\r\n")
      f.write("# translate every second line of this file.\r\n")
      if origMessages.messages[0]
        for i in 0...origMessages.messages[0].length
          msgs = origMessages.messages[0][i]
          Messages.writeObject(f, msgs, "Map#{i}", origMessages)
        end
      end
      for i in 1...origMessages.messages.length
        msgs = origMessages.messages[i]
        Messages.writeObject(f, msgs, i, origMessages)
      end
    }
  end

  def setMessages(type, array)
    @messages = [] if !@messages
    arr = []
    for i in 0...array.length
      arr[i] = (array[i]) ? array[i] : ""
    end
    @messages[type] = arr
  end

  def addMessages(type, array)
    @messages = [] if !@messages
    arr = (@messages[type]) ? @messages[type] : []
    for i in 0...array.length
      arr[i] = (array[i]) ? array[i] : (arr[i]) ? arr[i] : ""
    end
    @messages[type] = arr
  end

  def self.createHash(_type, array)
    arr = OrderedHash.new
    for i in 0...array.length
      if array[i]
        key = Messages.stringToKey(array[i])
        arr[key] = array[i]
      end
    end
    return arr
  end

  def self.addToHash(_type, array, hash)
    hash = OrderedHash.new if !hash
    for i in 0...array.length
      if array[i]
        key = Messages.stringToKey(array[i])
        hash[key] = array[i]
      end
    end
    return hash
  end

  def setMapMessagesAsHash(type, array)
    @messages = [] if !@messages
    @messages[0] = [] if !@messages[0]
    @messages[0][type] = Messages.createHash(type, array)
  end

  def addMapMessagesAsHash(type, array)
    @messages = [] if !@messages
    @messages[0] = [] if !@messages[0]
    @messages[0][type] = Messages.addToHash(type, array, @messages[0][type])
  end

  def setMessagesAsHash(type, array)
    @messages = [] if !@messages
    @messages[type] = Messages.createHash(type, array)
  end

  def addMessagesAsHash(type, array)
    @messages = [] if !@messages
    @messages[type] = Messages.addToHash(type, array, @messages[type])
  end

  def saveMessages(filename = nil)
    filename = "Data/messages.dat" if !filename
    File.open(filename, "wb") { |f| Marshal.dump(@messages, f) }
  end

  def loadMessageFile(filename)
    begin
      pbRgssOpen(filename, "rb") { |f| @messages = Marshal.load(f) }
      if !@messages.is_a?(Array)
        @messages = nil
        raise "Corrupted data"
      end
      return @messages
    rescue
      @messages = nil
      return nil
    end
  end

  def set(type, id, value)
    delayedLoad
    return if !@messages
    return if !@messages[type]
    @messages[type][id] = value
  end

  def getCount(type)
    delayedLoad
    return 0 if !@messages
    return 0 if !@messages[type]
    return @messages[type].length
  end

  def get(type, id)
    delayedLoad
    return "" if !@messages
    return "" if !@messages[type]
    return "" if !@messages[type][id]
    return @messages[type][id]
  end

  def getFromHash(type, key)
    delayedLoad
    return key if !@messages || !@messages[type] || !key
    id = Messages.stringToKey(key)
    return key if !@messages[type][id]
    return @messages[type][id]
  end

  def getFromMapHash(type, key)
    delayedLoad
    return key if !@messages
    return key if !@messages[0]
    return key if !@messages[0][type] && !@messages[0][0]
    id = Messages.stringToKey(key)
    if @messages[0][type] && @messages[0][type][id]
      return @messages[0][type][id]
    elsif @messages[0][0] && @messages[0][0][id]
      return @messages[0][0][id]
    end
    return key
  end
end

module MessageTypes
  # Value 0 is used for common event and map event text
  Species = 1
  Kinds = 2
  Entries = 3
  FormNames = 4
  Moves = 5
  MoveDescriptions = 6
  Items = 7
  ItemPlurals = 8
  ItemDescriptions = 9
  Abilities = 10
  AbilityDescs = 11
  Types = 12
  TrainerTypes = 13
  TrainerNames = 14
  BeginSpeech = 15
  EndSpeechWin = 16
  EndSpeechLose = 17
  RegionNames = 18
  PlaceNames = 19
  PlaceDescriptions = 20
  MapNames = 21
  PhoneMessages = 22
  TrainerLoseText = 23
  ScriptTexts = 24
  RibbonNames = 25
  RibbonDescriptions = 26
  @@messages = Messages.new
  @@messagesFallback = Messages.new("Data/messages.dat", true)

  def self.stringToKey(str)
    return Messages.stringToKey(str)
  end

  def self.normalizeValue(value)
    return Messages.normalizeValue(value)
  end

  def self.denormalizeValue(value)
    Messages.denormalizeValue(value)
  end

  def self.writeObject(f, msgs, secname)
    Messages.denormalizeValue(str)
  end

  def self.extract(outfile)
    @@messages.extract(outfile)
  end

  def self.setMessages(type, array)
    @@messages.setMessages(type, array)
  end

  def self.addMessages(type, array)
    @@messages.addMessages(type, array)
  end

  def self.createHash(type, array)
    Messages.createHash(type, array)
  end

  def self.addMapMessagesAsHash(type, array)
    @@messages.addMapMessagesAsHash(type, array)
  end

  def self.setMapMessagesAsHash(type, array)
    @@messages.setMapMessagesAsHash(type, array)
  end

  def self.addMessagesAsHash(type, array)
    @@messages.addMessagesAsHash(type, array)
  end

  def self.setMessagesAsHash(type, array)
    @@messages.setMessagesAsHash(type, array)
  end

  def self.saveMessages(filename = nil)
    @@messages.saveMessages(filename)
  end

  def self.loadMessageFile(filename)
    @@messages.loadMessageFile(filename)
  end

  def self.get(type, id)
    ret = @@messages.get(type, id)
    if ret == ""
      ret = @@messagesFallback.get(type, id)
    end
    return ret
  end

  def self.getCount(type)
    c1 = @@messages.getCount(type)
    c2 = @@messagesFallback.getCount(type)
    return c1 > c2 ? c1 : c2
  end

  def self.getOriginal(type, id)
    return @@messagesFallback.get(type, id)
  end

  def self.getFromHash(type, key)
    @@messages.getFromHash(type, key)
  end

  def self.getFromMapHash(type, key)
    @@messages.getFromMapHash(type, key)
  end
end

def pbLoadMessages(file)
  return MessageTypes.loadMessageFile(file)
end

def pbGetMessageCount(type)
  return MessageTypes.getCount(type)
end

def pbGetMessage(type, id)
  return MessageTypes.get(type, id)
end

def pbGetMessageFromHash(type, id)
  return MessageTypes.getFromHash(type, id)
end

# Replaces first argument with a localized version and formats the other
# parameters by replacing {1}, {2}, etc. with those placeholders.
def _INTL(*arg)
  begin
    string = MessageTypes.getFromHash(MessageTypes::ScriptTexts, arg[0])
  rescue
    string = arg[0]
  end
  string = string.clone
  for i in 1...arg.length
    string.gsub!(/\{#{i}\}/, "#{arg[i]}")
  end
  return string
end

# Replaces first argument with a localized version and formats the other
# parameters by replacing {1}, {2}, etc. with those placeholders.
# This version acts more like sprintf, supports e.g. {1:d} or {2:s}
def _ISPRINTF(*arg)
  begin
    string = MessageTypes.getFromHash(MessageTypes::ScriptTexts, arg[0])
  rescue
    string = arg[0]
  end
  string = string.clone
  for i in 1...arg.length
    string.gsub!(/\{#{i}\:([^\}]+?)\}/) { |m|
      next sprintf("%" + $1, arg[i])
    }
  end
  return string
end

def _I(str)
  return _MAPINTL($game_map.map_id, str)
end

def _MAPINTL(mapid, *arg)
  string = MessageTypes.getFromMapHash(mapid, arg[0])
  string = string.clone
  for i in 1...arg.length
    string.gsub!(/\{#{i}\}/, "#{arg[i]}")
  end
  return string
end

def _MAPISPRINTF(mapid, *arg)
  string = MessageTypes.getFromMapHash(mapid, arg[0])
  string = string.clone
  for i in 1...arg.length
    string.gsub!(/\{#{i}\:([^\}]+?)\}/) { |m|
      next sprintf("%" + $1, arg[i])
    }
  end
  return string
end