thattommyhall.com

current = ''
file_count = 0
total_size = 0

output = File.open('output.csv','w')

IO.popen('hadoop fs -lsr /user/hive/warehouse').each_line do |line|
  split = line.split(/\s+/)
  #permissions,replication,user,group,size,mod_date,mod_time,path
  next unless split.size == 8
  path = split[7]
  size = split[4]
  permissions = split[0]
  tablename=path.split('/')[4]
  if tablename != current
    average_size = file_count == 0 ? 0 : total_size/file_count
    result = "#{current},#{file_count},#{total_size},#{average_size}"
    unless current==''
      puts result
      output.puts result
    end
    total_size = 0
    current = tablename
    file_count = 0
  end
  file_count += 1 unless permissions[0] == 'd'
  total_size += size.to_i
end

limit nofile 20000 20000

kill timeout 300

env MONGO_DATA=/var/lib/mongodb/
env MONGO_LOGS=/var/log/mongodb/
env MONGO_EXE=/usr/bin/mongod
env MONGO_CONF=/etc/mongodb.conf

pre-start script
  mkdir -p $MONGO_DATA
  mkdir -p $MONGO_LOGS
  if [ -f $MONGO_DATA/mongod.lock ]; then
    mongo_pid=`cat $MONGO_DATA/mongod.lock`
    if [ ! -z $mongo_pid ]; then
      if [ ! `pgrep mongo | grep "$mongo_pid" | wc -l` -gt 0 ]; then
        rm $MONGO_DATA/mongod.lock
        sudo -u mongodb /usr/bin/mongod --config /etc/mongodb.conf --repair
        touch $MONGO_DATA/repaired-`date "+%Y%m%d-%H%M%S"`
      fi
    fi
  fi  
end script

start on runlevel [2345]
stop on runlevel [06]

script
  if [ -f /etc/default/mongodb ]; then . /etc/default/mongodb; fi
  exec start-stop-daemon --start --quiet --chuid mongodb --exec  $MONGO_EXE -- --config $MONGO_CONF
end script