#!/bin/bash

# Copyright (c) 2013 Cloudera, Inc. All rights reserved.

# debug
set -x

# Time marker for both stderr and stdout
date; date 1>&2

cloudera_config=`dirname $0`
cloudera_config=`cd "$cloudera_config/../common"; pwd`
. ${cloudera_config}/cloudera-config.sh

# Load the parcel environment
source_parcel_environment

# attempt to find java
locate_cdh_java_home

set_hive_classpath

HADOOP_CLIENT_OPTS=$(replace_pid $HADOOP_CLIENT_OPTS)

if [ "$CDH_VERSION" -ge "4" ]; then
  # Disable IPv6. This is automatically done by bin/hadoop for CDH3 clients.
  export HADOOP_CLIENT_OPTS="-Djava.net.preferIPv4Stack=true $HADOOP_CLIENT_OPTS"
fi

echo "using $JAVA_HOME as JAVA_HOME"
echo "using $CDH_VERSION as CDH_VERSION"

export HADOOP_HOME=$CDH_HADOOP_HOME
export HIVE_HOME=$CDH_HIVE_HOME
export SENTRY_HOME=$CDH_SENTRY_HOME
export HIVE_CONF_DIR=$CONF_DIR

SCHEMATOOL_DBTYPE=$HIVE_METASTORE_DATABASE_TYPE
# schematool uses different dbtype alias for postgres
if [[ "$SCHEMATOOL_DBTYPE" == postgresql ]]; then
  SCHEMATOOL_DBTYPE="postgres"
  # OPSAPS-16280 tell hive schematool to be compatible with postgres 8.1
  POSTGRESQL81_WORKAROUND="-dbOpts postgres.filter.81,postgres.filter.pre.9"
fi

function check_version_run_schematool_upgrade() {
  VERSION=$1
  # Minimum schema version supported for Oracle is 9, otherwise is 5.
  # Schema version -1 implies we couldn't figure out the schema version.
  if ([[ "$HIVE_METASTORE_DATABASE_TYPE" == oracle ]] && [[ "$VERSION" -lt "9" ]]) || [[ "$VERSION" -lt "5" ]] ; then
    echo "ERROR: Upgrade is not supported from current schema version for $HIVE_METASTORE_DATABASE_TYPE database type."
    exit 1
  fi
  if [[ "$VERSION" -ge "12" ]]; then
    # Starting schema version 12, HMS schema has the VERSION table
    exec bash -x $HIVE_HOME/bin/schematool -verbose -dbType $SCHEMATOOL_DBTYPE -upgradeSchema $POSTGRESQL81_WORKAROUND
  else
    VERSION_STR="0.$VERSION.0"
    exec bash -x $HIVE_HOME/bin/schematool -verbose -dbType $SCHEMATOOL_DBTYPE -upgradeSchemaFrom $VERSION_STR $POSTGRESQL81_WORKAROUND
  fi
}

# Add hive-hcatalog-server-extensions plugin jar in classpath.
set_classpath_for_plugin_jars() {
  if [ -z $1 ]; then
    echo "Must call with the name of variable to assign."
    exit 1
  fi

  # include hcatalog jar if asked by CM - INCLUDE_HIVE_HCATALOG_JAR=true
  if [ "$INCLUDE_HIVE_HCATALOG_JAR" = "true" ]; then
    # Here we find a jar for hive-hcatalog-server-extensions and add it to the given
    # environment variable.
    if [[ -n "$CDH_HCAT_HOME" ]]; then
      # Add hive-hcatalog-server-extensions plugin jar to the classpath.
      ADD_TO_CP=`find "${CDH_HCAT_HOME}/share/hcatalog/" -maxdepth 1 -name 'hive-hcatalog-server-extensions*.jar' | tr "\n" ":"`
      eval OLD_VALUE=\$$1
      NEW_VALUE="$ADD_TO_CP$OLD_VALUE"
      export $1=${NEW_VALUE/%:/}  # Remove trailing ':' if present.
    else
      echo "CDH_HCAT_HOME must be set."
      exit 1
    fi
  fi
}

# Check yarn-conf first because HDFS could also generate hadoop-conf.
if [ -d $CONF_DIR/yarn-conf ]; then
  export HADOOP_CONF_DIR=$CONF_DIR/yarn-conf
  # (See CDH-10123) Hive doesn't use HADOOP_MAPRED_HOME to figure out
  # whether it should use MR1 or 2.
  # So we need to add its contents to classpath manually.
  # Also exporting it so that if it gets fixed in future, we don't have to do anything.
  export HADOOP_MAPRED_HOME=$CDH_MR2_HOME
elif [ -d $CONF_DIR/hadoop-conf ]; then
  export HADOOP_CONF_DIR=$CONF_DIR/hadoop-conf
  # in CDH5, hive prefers MR2. To be safe, always set HADOOP_MAPRED_HOME
  export HADOOP_MAPRED_HOME=$CDH_MR1_HOME
else
  echo "No config directory found for MR."
  exit 1
fi
# Add contents of HADOOP_MAPRED_HOME to classpath manually
if [ -d $HADOOP_MAPRED_HOME ]; then
  for i in "$HADOOP_MAPRED_HOME/"*.jar; do
    AUX_CLASSPATH="${AUX_CLASSPATH}:$i"
  done
fi

set_classpath_for_plugin_jars AUX_CLASSPATH

echo "using $HIVE_HOME as HIVE_HOME"
echo "using $HIVE_CONF_DIR as HIVE_CONF_DIR"
echo "using $HADOOP_HOME as HADOOP_HOME"
echo "using $HADOOP_CONF_DIR as HADOOP_CONF_DIR"

# Search-replace {{CMF_CONF_DIR}} in files
replace_conf_dir

replace_hive_hbase_jars_template "hive-site.xml"

JDBC_JARS="$CLOUDERA_MYSQL_CONNECTOR_JAR:$CLOUDERA_POSTGRESQL_JDBC_JAR:$CLOUDERA_ORACLE_CONNECTOR_JAR"
if [[ -z "$AUX_CLASSPATH" ]]; then
  export AUX_CLASSPATH="$JDBC_JARS"
else
  export AUX_CLASSPATH="$AUX_CLASSPATH:$JDBC_JARS"
fi

if ([[ -n "$HIVE_AUX_JARS_PATH" ]] && [[ ! -d "$HIVE_AUX_JARS_PATH" ]]); then
  $( cat <<EOF
ERROR: HIVE_AUX_JARS_PATH is configured in Cloudera Manager as $HIVE_AUX_JARS_PATH. 
However directory $HIVE_AUX_JARS_PATH does not exist. When configured, directory specified 
in HIVE_AUX_JARS_PATH must be created and managed manually before starting Hive.
EOF
)
  exit 1
fi

if [[ "$1" == webhcat ]]; then
  export HCAT_PREFIX="$CDH_HCAT_HOME"
  export WEBHCAT_CONF_DIR="$CONF_DIR"
  export HADOOP_PREFIX="$HADOOP_HOME"
  export HADOOP_LIBEXEC_DIR="$CDH_HADOOP_HOME/libexec/"
  export WEBHCAT_LOG_DIR="$HIVE_LOG_DIR"
  export WEBHCAT_LOG4J="file:$WEBHCAT_CONF_DIR/webhcat-log4j.properties"
  echo "using $HCAT_PREFIX as HCAT_PREFIX"
  echo "using $WEBHCAT_CONF_DIR as WEBHCAT_CONF_DIR"
  echo "using $HADOOP_PREFIX as HADOOP_PREFIX"
  echo "using $JAVA_HOME as JAVA_HOME"
  echo "using $HADOOP_LIBEXEC_DIR as HADOOP_LIBEXEC_DIR"
  echo "using $WEBHCAT_LOG_DIR as WEBHCAT_LOG_DIR"
  echo "using $WEBHCAT_LOG4J as WEBHCAT_LOG4J"
  WEBHCAT_SERVER_CMD="$HCAT_PREFIX/sbin/webhcat_server.sh --config \"$WEBHCAT_CONF_DIR\" foreground"

  # Find the correct webhcat-default.xml
  if [ ! -e "$WEBHCAT_DEFAULT_XML" ]; then
    if [[ -n "$PARCELS_ROOT" ]]; then
      # try CDH4.3+ and CDH4.2.1 parcel dirs
      for CANDIDATE in "$PARCELS_ROOT/CDH/etc/"{hive-,}"webhcat/conf.dist/webhcat-default.xml"
      do
        if [ -e "$CANDIDATE" ]; then
          WEBHCAT_DEFAULT_XML="$CANDIDATE"
          break
        fi
      done
    else
      # try CDH4.2.x packages dir
      CANDIDATE="/etc/webhcat/conf.dist/webhcat-default.xml"
      if [ -e "$CANDIDATE" ]; then
          WEBHCAT_DEFAULT_XML="$CANDIDATE"
      fi
    fi
  fi

  # Copy webhcat-default.xml to conf dir
  echo "WEBHCAT_DEFAULT_XML=$WEBHCAT_DEFAULT_XML"
  if !(cp "$WEBHCAT_DEFAULT_XML" "$WEBHCAT_CONF_DIR/"); then
    echo "ERROR: failed to copy webhcat-default.xml"
    exit 1
  fi

  # webhcat-default.xml references PYTHON_CMD and it will die without it
  export PYTHON_CMD="$(which python)"

  if [ -f "$HCAT_PREFIX/share/webhcat/java-client/webhcat-java-client-0.4.0-cdh4.2.0.jar" ]; then
    # CDH4.2.0 has a buggy webhcat_server.sh that fails to kill the real webhcat server
    # trap SIGTERM to nothing, forcing supervisor to do a group SIGKILL, which will correctly clean up all processes
    trap "" SIGTERM
    $WEBHCAT_SERVER_CMD
    # this process is always killed, shouldn't exit here unless WebHCat dies abnormally
    exit $?
  else
    # CDH4.2.1+ webhcat_server.sh does not require special kill handling
    exec $WEBHCAT_SERVER_CMD
  fi

elif [[ "$1" == hiveserver2 ]]; then
  if [[ "$SPARK_ON_YARN" == "true" ]]; then
    # Add spark-defaults.properties from spark client config to hive conf dir.
    # Ideally this would be placed in the conf dir by CM, but see OPSAPS-25660.
    # We use a symlink so that if HS2 starts before you've first deployed spark
    # CC, then things still work. HS2 loads spark-defaults.properties each time
    # a client opens a spark session.
    export SPARK_CONF_DIR="${SPARK_CONF_DIR:-/etc/spark/conf}"
    echo "using $SPARK_CONF_DIR as SPARK_CONF_DIR"
    SPARK_FILE="$SPARK_CONF_DIR/spark-defaults.conf"
    if [[ ! -e "$SPARK_FILE" ]]; then
      echo "WARNING: No spark-defaults.properties at $SPARK_FILE. Ensure there is a SPARK_ON_YARN role (such as a Gateway) on this HiveServer2 host and you have deployed Spark's client configuration before running Hive on Spark jobs."
    fi
    if [[ -e "$CONF_DIR/spark-defaults.conf" ]]; then
      echo "WARN: spark-defaults.conf already exists in $CONF_DIR. Skipping symlink creation."
    else
      ln -s "$SPARK_FILE" "$CONF_DIR/spark-defaults.conf"
    fi
  fi
  # check for job secrets keystore to upload
  if [[ -f "$CONF_DIR/$HIVE_JOB_CREDSTORE_SRC" ]]; then
    dir=$(dirname "$HIVE_JOB_CREDSTORE_DEST")
    if [ ! $(hadoop fs -test -d "$dir") ]; then
      hadoop fs -mkdir "$dir"
      hadoop fs -chmod 700 "$dir"
    fi
    hadoop fs -put "$CONF_DIR/$HIVE_JOB_CREDSTORE_SRC" "$HIVE_JOB_CREDSTORE_DEST"
    hadoop fs -chmod 600 "$HIVE_JOB_CREDSTORE_DEST"
  fi

elif [[ "$1" == updatelocation ]]; then
  NEW_LOC="$2"
  echo "Updating Hive Metastore to use location $NEW_LOC"
  FS_ROOT_OUTPUT=`$HIVE_HOME/bin/hive --config $CONF_DIR --service metatool -listFSRoot`
  FS_ROOT_RETURN=$?
  echo "List FS Root output:"
  echo $FS_ROOT_OUTPUT
  # last line of output is the fs root
  FS_ROOT=$(echo "$FS_ROOT_OUTPUT" | tail -n 1)

  # make sure we got a legal hdfs location and not some error
  if [[ 0 == $FS_ROOT_RETURN && "$FS_ROOT" =~ ^hdfs://.* ]] ; then
    echo "Old FS Root: $FS_ROOT"
  else
    echo "ERROR: Failed while listing FS Root."
    exit 1
  fi

  # FS_ROOT=hdfs://oldhost:1234/path/to/warehouse       where 1234 is the port
  # split on / to get host and port "oldhost:1234"
  OLD_LOC_WITH_PORT=$(echo "$FS_ROOT" | awk '{split($0,array,"/")} END {print array[3]}')
  # add back "hdfs://" prefix
  OLD_LOC_WITH_PORT="hdfs://$OLD_LOC_WITH_PORT"

  # strip out port
  OLD_LOC=$(echo "$OLD_LOC_WITH_PORT" | awk '{split($0,array,":")} END {print array[2]}')
  # add back "hdfs:" prefix
  OLD_LOC="hdfs:$OLD_LOC"

  if [[ "$NEW_LOC" == "$OLD_LOC" ]]; then
    echo "FS Root already matches $NEW_LOC. No Update Required"
    exit 0
  fi

  # Run the command twice; first with including the port number
  # and then without the port number. This would take care of
  # both instances where the schema-urls contain or do not contain port
  # The second command would also take care of pre-HIVE-3665 versions
  $HIVE_HOME/bin/hive --config $CONF_DIR --service metatool -updateLocation "$NEW_LOC" "$OLD_LOC_WITH_PORT" -tablePropKey avro.schema.url -serdePropKey avro.schema.url
  exec $HIVE_HOME/bin/hive --config $CONF_DIR --service metatool -updateLocation "$NEW_LOC" "$OLD_LOC" -tablePropKey avro.schema.url -serdePropKey avro.schema.url

elif [[ "$1" == create_metastore_tables ||  "$1" == upgrade_metastore ]]; then
  if [[ -z "$CMF_SERVER_ROOT" ]]; then
    JDBC_JARS_CLASSPATH="/usr/share/cmf/lib/*:$JDBC_JARS"
  else
    JDBC_JARS_CLASSPATH="$CMF_SERVER_ROOT/lib/*:$JDBC_JARS"
  fi
  if [[ "$1" == create_metastore_tables ]]; then
    if [ "$CDH_VERSION" -ge "5" ]; then
      # CDH 5 uses schema tool to create schema, but it isn't idempotent
      skip_if_tables_exist metastore_db_py.properties

      # hive config is available to schematool via env var HIVE_CONF_DIR
      exec bash -x $HIVE_HOME/bin/schematool -verbose -dbType $SCHEMATOOL_DBTYPE -initSchema $POSTGRESQL81_WORKAROUND
    else
      exec $JAVA_HOME/bin/java -Djava.net.preferIPv4Stack=true -cp "$JDBC_JARS_CLASSPATH" com.cloudera.cmf.service.hive.HiveMetastoreDbUtil "$CONF_DIR/metastore_db_py.properties" "$(dirname $0)/ddl" "--createTables"
    fi
  elif [[ "$1" == upgrade_metastore ]]; then
    VERSION=$($JAVA_HOME/bin/java -Djava.net.preferIPv4Stack=true -cp "$JDBC_JARS_CLASSPATH" com.cloudera.cmf.service.hive.HiveMetastoreDbUtil "$CONF_DIR/metastore_db_py.properties" "" "--printSchemaVersion")
    check_version_run_schematool_upgrade $VERSION
  fi
elif [[ "$1" == validate_metastore ]]; then
  exec bash -x $HIVE_HOME/bin/schematool -verbose -validate -dbType $SCHEMATOOL_DBTYPE $POSTGRESQL81_WORKAROUND
fi
exec $HIVE_HOME/bin/hive --config $CONF_DIR --service "$@"
