/* ***************************************************************************

*
* Module PXBridgeHelper.java
*
* IBM Confidential
* OCO Source Materials
* (c) Copyright IBM Corp. 2001, 2014, 2017, 2018
*
* (c) Copyright 2009, 2014 IBM Corporation - All Rights Reserved
* This is unpublished proprietary source code of IBM Corporation
* The copyright notice above does not evidence any actual or intended
* publication of such source code.
*
* Maintenance log - insert most recent change descriptions at top
*
* Date....... ECASE WHO Description..................................
* Sep 16, 2009 ----- lmahanam Changes to support Push Down Optimization
* Aug 13, 2009 ----- cedirisu Multi-Connector OSH support changes
* Aug 03, 2009 ----- cedirisu Job Runtime OSH generation Refactor changes
* Jun 01, 2009 ----- cedirisu Initial Creation
*
***************************************************************************** */
package com.ascential.investigate.utils.jobs.osh.operator;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import com.ascential.investigate.exception.SorcererException;
import com.ascential.investigate.shared.logging.SorcererServiceLogging;
import com.ascential.investigate.utils.XMETAHelper;
import com.ascential.investigate.utils.dbutils.DataConnection;
import com.ascential.investigate.utils.dbutils.DatabaseTypeSourceEnum;
import com.ascential.investigate.utils.dbutils.DatabaseTypeUtils;
import
com.ascential.investigate.utils.jobs.osh.connector.cpg.descriptor.PropertyDefinitio
n;
import com.ascential.investigate.utils.jobs.osh.payload.ColumnInfo;
import com.ascential.investigate.utils.jobs.osh.payload.ExecutionEngineInfo;
import com.ascential.investigate.utils.jobs.osh.payload.SQLVirtualTableInfo;
import com.ascential.investigate.utils.jobs.osh.payload.TableInfo;
import com.ascential.investigate.utils.jobs.properties.JobProperties;
import com.ascential.investigate.utils.model.IAModelUtils;
import com.ibm.infosphere.ia.utils.connectors.TableAction;
import com.ibm.infosphere.ia.utils.connectors.WriteMode;

import ASCLModel.investigate.CredentialMapping;

/**
* Helper class used to formulate data for the pxbridge osh
*
* @author Chamath Edirisuriya
*
*/
public class PXBridgeHelper implements Serializable {
 private static final long serialVersionUID = 1L;

private static final String WHERECLAUSE_MANUAL_EDIT_PREFIX = "/*where clause

edited*/";

protected int bridgeContext_;

protected String pxb_tableName_ = "";

protected String pxb_whereClause_ = "";
protected int pxb_inputDataFieldNumber_ = 0;
protected boolean pxb_compressionEnabled = false;

protected ArrayList pxb_sqlStmtColumns_ = null;

protected ArrayList pxb_sqlStmtAliasColumns_ = null;
protected ArrayList pxb_nullableColumns_ = null;
protected ArrayList pxb_columnPXTypes_ = null;
protected ArrayList pxb_columnDataTypes_ = null;
protected ArrayList pxb_columnNativeTypes_ = null;
protected List pxb_selectedSQLExpressions_ = null;
protected String pxb_recordCount_ = "2000";
protected String pxb_arraySize_ = "1";
protected String pxb_batchSize_ = "2000";
protected String pxb_isolationLevel_ = "1";
protected String pxb_ignoreSizeChecks_ = "0";
protected String pxb_passLob_ = "0";
protected String pxb_codePage_ = "0";
protected String pxb_generateSql_ = "0";
protected String pxb_ignoreTypeChecks_ = "0";
protected String pxb_autoCommit_ = "0";
protected String pxb_EndOfWave_ = "0";
protected String pxb_selectStatement_ = null;
protected String pxb_insertStatement_ = null;
protected String pxb_updateStatement_ = null;
protected String pxb_beforeStatement = null;
protected String pxb_afterStatement = null;
protected String pxb_createStatement_ = null;
protected String pxb_createStatementFailOnError_ = "0";
protected String pxb_generateCreateStatement_ = "0";
protected String pxb_generateTruncateStatement_ = "0";

private String tableSpace = null;

protected List<TableAction> tableActionList_ = null;
protected TableAction tableAction = TableAction.append;
protected List<WriteMode> writeModeList_ = null;
protected WriteMode writeMode = WriteMode.insert;

protected ExecutionEngineInfo engineInfo_ = null;

protected List<TableInfo> tableInfoList_ = null;
protected TableInfo tableInfo_ = null;
protected DataConnection dataConnection_ = null;
private Map<String, String> properties = new HashMap<String, String>();
//Microseconds added for timestamp and time data type in DS Schema
public static final String MICROSECONDS = "microseconds";
private int indexIntializer = 0;
private JobProperties jobProperties;
protected String projectRid;

public PXBridgeHelper(int bridgeContext, ExecutionEngineInfo engineInfo,

List<TableInfo> tableInfoList,
DataConnection dataConnection, String tableSpace,
List<TableAction> tableActionList, List<WriteMode> writeModeList) throws
SorcererException
{
initialize(bridgeContext, engineInfo, tableInfoList, dataConnection,
tableSpace, tableActionList, writeModeList);
}

public PXBridgeHelper(int bridgeContext, ExecutionEngineInfo engineInfo,

JobProperties jobProperties, List<TableInfo> tableInfoList,
DataConnection dataConnection, String tableSpace,
List<TableAction> tableActionList, List<WriteMode> writeModeList) throws
SorcererException
{
initialize(bridgeContext, engineInfo, tableInfoList,
dataConnection, tableSpace, tableActionList, writeModeList);
if( null != jobProperties )
{
this.jobProperties = jobProperties;
this.projectRid = jobProperties.getProjectRID();
}
}

public void initialize(int bridgeContext, ExecutionEngineInfo engineInfo,

List<TableInfo> tableInfoList,
DataConnection dataConnection, String tableSpace,
List<TableAction> tableActionList, List<WriteMode> writeModeList) throws
SorcererException
{
bridgeContext_ = bridgeContext;
engineInfo_ = engineInfo;
tableInfoList_ = tableInfoList;
this.tableSpace = tableSpace;
this.tableActionList_ = tableActionList;
this.writeModeList_ = writeModeList;
this.dataConnection_ = dataConnection;

// For the first initialization, set 'tableInfo_' to first element of

tableInfoList
if( null != tableInfoList_ )
{
tableInfo_ = tableInfoList_.get(0);
}

if( null != tableActionList_ && !tableActionList_.isEmpty() )

{
tableAction = tableActionList_.get(0);
}

if( null != writeModeList_ && !writeModeList_.isEmpty() )

{
writeMode = writeModeList_.get(0);
}

if ( dataConnection_ != null ) {

// add the connection properties

Map<String, String> connectionParameters =
dataConnection_.getParameters();

XMETAHelper xmetaHelper = new XMETAHelper();

CredentialMapping credentialMapping = null;
try
{
if (projectRid==null)
{
projectRid =
IAModelUtils.getProjectRIDOfTableAnalysisMaster(tableInfo_.getTableRID(),
xmetaHelper);
}
if(projectRid != null)
{
credentialMapping =
IAModelUtils.getCredentialMapping(projectRid, dataConnection_.getRID(),
xmetaHelper);
if(credentialMapping == null)
{
credentialMapping = IAModelUtils.getCredentialMapping(null,
dataConnection_.getRID(), xmetaHelper);
}
}
}finally
{
xmetaHelper.release();
}

for (String paramName : connectionParameters.keySet())

{
String paramValue = connectionParameters.get(paramName);

if(credentialMapping != null)
{
if(paramName.equals("Username"))
{
paramValue = credentialMapping.getUser();
}

if(paramName.equals("Password"))
{
paramValue = credentialMapping.getPassword();
}
}
if (paramValue!=null) {
properties.put(paramName, paramValue);
}
}
SorcererServiceLogging.debug("parameters: " + properties);
}

updateProperties();

if ( engineInfo_ != null ){

setAutoCommitMode(String.valueOf(engineInfo_.getAutoCommitMode()));
setIsolationLevel(engineInfo_.getIsolationLevel());
setArraySize(engineInfo_.getArraySize());
}
 }

public void reInitialize()

{
indexIntializer++;

if( null != tableInfoList_ )

{
tableInfo_ = tableInfoList_.get(indexIntializer);
}
else {
tableInfo_ = null;
return;
}

if( null != tableActionList_ )

{
tableAction = tableActionList_.get(indexIntializer);
}
else {
tableAction = TableAction.append;
}

if( null != writeModeList_ && !writeModeList_.isEmpty() )

{
writeMode = writeModeList_.get(indexIntializer);
}
else {
writeMode = WriteMode.insert;
}

updateProperties();
}

public void setAutocommit(boolean autocommit)

{
this.pxb_autoCommit_ = autocommit ? "1" : "0";
}

private void updateProperties()

{
if ( tableInfo_ != null ){
pxb_tableName_ = tableInfo_.getTableName();
pxb_inputDataFieldNumber_ = tableInfo_.getInputDataFieldNumber();
pxb_whereClause_ =
cleanseWhereClause(tableInfo_.getWhereClause());
pxb_compressionEnabled = tableInfo_.isCompressionEnabled();
}

setFieldInfo(tableInfo_);
}

public int nbOfTargetTables()

{
if( bridgeContext_ == PXBridgeOperator.CONTEXT_TARGET )
{
return tableInfoList_.size();
}
return 0;
 }

/**
* IA 8.1.1 and earlier stored in XMeta where clauses with escaped quotes.
(For instance: "where col=\'x\'"
* IA 8.1.2 and later store in XMeta non escaped where clause (For instance:
"where col='x'". The escape is done during the job generation.
* When 8.1.1 is upgraded to 8.1.2 or later, CA and rule execution would fail
with where conditions defined in 8.1.1 or earlier
* because they don't expect the string to be escaped and produce a double
escape during the job generation.
* Therefore we need to look at the where clause before using is and see if
it was generated by 8.1.1- or 8.1.2+.
* This function takes a where clause and remove the 8.1.1 escape chars if
necessary to return a non escape string as if it had
* been saved by 8.1.2+
* @param whereClause The where clause to cleanse
* @return The new where clause with 8.1.2 style
*/
private static String cleanseWhereClause(String whereClause)
{
if (whereClause==null) return null;

// If where clause has 'edit prefix', then remove it.

if (whereClause.indexOf(WHERECLAUSE_MANUAL_EDIT_PREFIX) >= 0) {
whereClause =
whereClause.substring(WHERECLAUSE_MANUAL_EDIT_PREFIX.length());
}

// if the clause doesn't contain any instance of \' then it is ok and

nothing needs to be done.
int firstEscapedQuoteIndex = whereClause.indexOf("\\'");
if (firstEscapedQuoteIndex<0) return whereClause;

// if the clause contains instances of \' we must look if it is a

clause generated by 8.1.1 with the escaped quotes
// or if it is a normal escaped char within a string literal generated
by 8.1.2+
int firstQuoteIndex = whereClause.indexOf("'");
// if we find a non escaped single quote before the first instance of
the escaped single quote, it means
// that the where clause is generally not escaped and that the escaped
quote we found is simply part of a string litteral
// In that case everything is ok.
if (firstQuoteIndex>=0 && firstQuoteIndex<firstEscapedQuoteIndex)
return whereClause;
// if there is no non-escaped quote before the first escaped quote, it
means that all quotes are escaped
// (where clause was generated by 8.1.1 or earlier). In that case, we
remove all the escape chars
else return whereClause.replaceAll("\\\\'", "'");
}

public String getTableSpace() {

return tableSpace;
}

public void setTableSpace(String tableSpace) {

this.tableSpace = tableSpace;
 }

/**
* check to see if the database has any lob types
* @return
*/
private boolean hasLob(){
if ( pxb_columnNativeTypes_ == null ) return false;

for ( int i = 0; i < pxb_columnNativeTypes_.size(); i++ ){

String nativeType = (String)pxb_columnNativeTypes_.get(i);
if ( nativeType != null ){
nativeType = nativeType.toUpperCase();
if (nativeType.indexOf("LOB")>=0 ||
nativeType.indexOf("TEXT")>=0 || nativeType.indexOf("XML")>=0
|| nativeType.indexOf("VARCHAR(MAX)")>=0 ||
nativeType.indexOf("VARCHAR(-1)")>=0)
{
return true;
}
// For FOXPRO native clob type
if ( nativeType.equals("MEMO")){
return true;
}
}
}

return false;
}

public void setArraySize(int arraySize) {

// If LOB type, set arraysize to 1

if (hasLob()){
pxb_arraySize_ = "1";
return;
}

// Always set arraysize to 1 for ACCESS

DatabaseTypeSourceEnum databaseType = getDatabaseType();
if (DatabaseTypeSourceEnum.ACCESS.equals(databaseType)) {
pxb_arraySize_ = "1";
return;
}

if (DatabaseTypeSourceEnum.ORACLE.equals(databaseType)) {
pxb_arraySize_ = "2000";
return;
}

if (DatabaseTypeSourceEnum.HIVE.equals(databaseType)) {
pxb_batchSize_ = new Integer(arraySize).toString();
if (this.bridgeContext_==PXBridgeOperator.CONTEXT_TARGET) {
pxb_arraySize_ = "1";
return;
}
}
// // If doing an UpSert, set to 1
// if (this.bridgeContext_==PXBridgeOperator.CONTEXT_TARGET &&
this.writeMode==WriteMode.update_then_insert) {
// pxb_arraySize_ = "1";
// return;
// }

pxb_arraySize_ = new Integer(arraySize).toString();

int recordCount = Integer.parseInt(pxb_recordCount_);

if (arraySize > recordCount) {
pxb_recordCount_ = pxb_arraySize_;
}
}

public void setGenerateSql(boolean set) {

pxb_generateSql_ = (set) ? "1" : "0";
}

protected void setAutoCommitMode(String autoCommitMode) {

DatabaseTypeSourceEnum databaseType = getDatabaseType();
if(DatabaseTypeSourceEnum.HIVE.equals(databaseType))
{
pxb_autoCommit_ = "0";
return;
}
pxb_autoCommit_ = autoCommitMode;
}

/**
* Here are Isolation levels for ODBCConnector: Read uncommitted = 1, Read
committed = 2, Repeatable read = 3, Serializable = 4
* @param isolationLevel
*/
public void setIsolationLevel(int isolationLevel) {

String connectorName = getConnectorName();

// isolationLevel values are different for different connectors
if(connectorName != null)
{
DatabaseTypeSourceEnum databaseType = getDatabaseType();
if(connectorName.contains("DB2Connector"))
{
switch (isolationLevel)
{
case 1 :
isolationLevel = 0; // Read uncommitted
break;
case 2 :
isolationLevel = 1; // Cursor stability - user
selection 2 is treated as this isolation level (custom mapping)
break;
case 3 :
isolationLevel = 3; // Repeatable read
break;
case 4 :
isolationLevel = 2; // Read stability - user
selection 4 is treated as this isolation level (custom mapping)
break;
default:
isolationLevel = 0; // // Read uncommitted
 break;
}
}
else if(connectorName.contains("OracleConnector"))
{
switch (isolationLevel)
{
case 2 :
isolationLevel = 0; // Read committed
break;
case 3 :
isolationLevel = 3; // Read only - user selection 3
is treated as this isolation level (custom mapping)
break;
case 4 :
isolationLevel = 1; // Serializable
break;
default:
isolationLevel = 0; // Read committed
break;
}
}
else if(connectorName.contains("TeradataConnector"))
{
switch (isolationLevel)
{
case 1 :
isolationLevel = 2; // Read uncommitted
break;
case 2 :
isolationLevel = 3; // Read committed
break;
case 3 :
isolationLevel = 4; // Repeatable read
break;
case 4 :
isolationLevel = 5; // Serializable
break;
default:
isolationLevel = 2; // Read uncommitted
break;
}
}
else if(connectorName.contains("JDBCConnector"))
{
if(DatabaseTypeSourceEnum.ORACLE.equals(databaseType) &&
isolationLevel == 1)
{
isolationLevel = 2; // Read uncommitted
}
else if(DatabaseTypeSourceEnum.HIVE.equals(databaseType))
{
isolationLevel = 0; // None
}
}
else if(connectorName.contains("HiveConnector"))
{
isolationLevel = 0; // None
}
 pxb_isolationLevel_ = String.valueOf(isolationLevel);
if (DatabaseTypeSourceEnum.SYBASE_SERVER.equals(databaseType)) {
pxb_isolationLevel_ = "0";
}
}
}

public int getType() {

return bridgeContext_;
}

public DatabaseTypeSourceEnum getDatabaseType()

{
return (dataConnection_!=null) ? dataConnection_.getDataSourceType() :
null;
}

public String getConnectorName()

{
return (dataConnection_!=null && dataConnection_.getConnector()!
=null) ? dataConnection_.getConnector().getName() : null;
}

public String getConnectorVariant()

{
return (dataConnection_!=null && dataConnection_.getConnector()!
=null) ? dataConnection_.getConnector().getVariant() : null;
}

public String getDSSchema() {

StringBuffer schema = new StringBuffer();
StringBuffer dsSchema = new StringBuffer();
List<String> clobColumnsList = new ArrayList<String>();

if (bridgeContext_ == PXBridgeOperator.CONTEXT_SOURCE) {
schema.append("-source 0 '{");
} else {
schema.append("-target ").append(indexIntializer).append(" '{");
}

schema.append("\n");
dsSchema.append(" DSSchema=\\'record (");
dsSchema.append("\n");

int numOfFields = 0;
if ( pxb_sqlStmtColumns_ != null ){
numOfFields = pxb_sqlStmtColumns_.size();
}

for (int i = 0; i < numOfFields; i++) {

String sqlName = (String) pxb_sqlStmtColumns_.get(i);
// see if the column name was aliased to remove wild characters
String fieldName = (String) pxb_sqlStmtAliasColumns_.get(i);
if ((fieldName == null) || (fieldName.length() == 0)) {
//
// If field name was passed in as alias, only want alias name in
// schema
int index = sqlName.indexOf(" as ", 0);
if (index > -1) {
 fieldName = sqlName.substring(index + 4);
} else {
fieldName = sqlName;
}
}

dsSchema.append(fieldName + ": ");

if (!pxb_nullableColumns_.get(i).equals("")) {
dsSchema.append((String) ("nullable "));
}
String pxType = (String)pxb_columnPXTypes_.get(i);
if(pxTypeisTimeOrTimeStamp(pxType)){
dsSchema.append(pxType+"\\["+MICROSECONDS+"\\]\\" + ";");
} else {
if(bridgeContext_ == PXBridgeOperator.CONTEXT_SOURCE &&
"CLOB".equals(pxb_columnNativeTypes_.get(i)))
{
clobColumnsList.add(fieldName);
dsSchema.append("ustring;");
}
else
{
dsSchema.append(pxType + ";");
}
}
dsSchema.append("\n");
}

// append the virtual columns expressed as SQL expressions

if (pxb_selectedSQLExpressions_ != null) {
ColumnInfo col = null;
for (Iterator sqlExpressions = pxb_selectedSQLExpressions_.iterator();
sqlExpressions.hasNext();) {
col = (ColumnInfo) sqlExpressions.next();
String fieldName = col.getColumnAlias();
if (fieldName==null) fieldName = col.getColumnName();
dsSchema.append(fieldName);
dsSchema.append(": ");
dsSchema.append(col.getPxMetaFormat());
dsSchema.append(";\n");
}
}

dsSchema.append(")\\'");
dsSchema.append("\n");
dsSchema.append("}'");

int size = clobColumnsList.size();

if(size > 0)
{
StringBuffer dsSQLType = new StringBuffer(" DSSQLType={");
for( int iIndex = 0; iIndex < size; iIndex++ )
{
dsSQLType.append(clobColumnsList.get(iIndex));
dsSQLType.append("=-1");
if(iIndex < size-1)
{
dsSQLType.append(",");
}
 }
dsSQLType.append("},");
dsSQLType.append("\n");
schema.append(dsSQLType.toString());
}

schema.append(dsSchema.toString());
return schema.toString();
}

/**
* Checks the type of datatype based on database type enum
* @param pxType
* @return
*/
private boolean pxTypeisTimeOrTimeStamp(String pxType) {
if(pxType!= null && (pxType.equals("time") ||
pxType.equals("timestamp"))){
return true;
}
return false;
}

public Map<String, String> getXMLPropertyValues() throws SorcererException{

if (bridgeContext_ == PropertyDefinition.CONTEXT_SOURCE) {
generateSelectStatement();

addSourceProperties();

} else if (bridgeContext_ == PropertyDefinition.CONTEXT_TARGET) {

generateCreateStatement();
generateInsertStatement();
generateUpdateStatement();
generateBeforeAfterSQL();

addTargetProperties();
}

return properties;
}

public String getXMLProperties()

{
StringBuffer xmlProperties = new StringBuffer();
xmlProperties.append("<Common><Context
type=\\'int\\'>").append(bridgeContext_).append("</Context>");
xmlProperties.append("<Variant
type=\\'string\\'>").append(getConnectorVariant()).append("</Variant>" );
xmlProperties.append("</Common>\n");

xmlProperties.append("<Usage>\n<Session>")
.append("<IsolationLevel type=\\'int\\'><![CDATA[")
.append(pxb_isolationLevel_).append("]]></IsolationLe
vel>\n")
.append("<AutocommitMode type=\\'int\\'><![CDATA[")
.append(pxb_autoCommit_).append("]]></AutocommitMode>
</Session>\n")
 .append("<RecordOrdering collapsed=\\'1\\'
type=\\'int\\'><![CDATA[0]]></RecordOrdering>\n</Usage>\n");

return xmlProperties.toString();
}

protected void generateSelectStatement() throws SorcererException

{
pxb_selectStatement_ =
generateSelectStatement(generateParsedSelectStatement());
SorcererServiceLogging.debug("select statement to be executed as part
of this PX Job is: " + pxb_selectStatement_);
}

protected String generateSelectStatement(SelectStatement

parsedSelectStatement)
{
if (parsedSelectStatement==null) return null;

StringBuffer selectStmt = new StringBuffer("select \n");

for (Iterator<SelectedField> selectedFields =
parsedSelectStatement.getSelectedFields().iterator() ; selectedFields.hasNext() ; )
{
SelectedField selectedField = selectedFields.next();
if(selectedField.getExpression().equalsIgnoreCase("NULL"))
{
selectStmt.append( "\"NULL\"");
}
else{
selectStmt.append(selectedField.getExpression());
}
String alias = selectedField.getAlias();
if (alias!=null) selectStmt.append(" as ").append(alias);
if (selectedFields.hasNext()) selectStmt.append(", ");
selectStmt.append("\n");
}
if(tableInfo_ instanceof SQLVirtualTableInfo){
selectStmt.append(" from
(").append(((SQLVirtualTableInfo)tableInfo_).getSelectStatement()).append(") A");
}else{
selectStmt.append(" from
").append(parsedSelectStatement.getSelectedTable());
String whereClause = parsedSelectStatement.getWhereClause();
if (whereClause!=null) selectStmt.append(" where
").append(whereClause);
}
if("NetezzaConnector".equalsIgnoreCase(getConnectorName())){
if(jobProperties != null && jobProperties.isSampleUsed())
{
DataSampleOperator sample = new
DataSampleOperator(jobProperties.getSampleOptions());
int maxRecords = sample.getMaximumRecords();
selectStmt.append(" limit ").append(maxRecords);
}
}
return selectStmt.toString();
}

protected SelectStatement generateParsedSelectStatement() throws

SorcererException{

if (pxb_selectStatement_ != null) {
return null;
}

DatabaseTypeSourceEnum databaseType = getDatabaseType();

SelectStatement selectStatement = new SelectStatement();

int numOfFields = 0;
if (pxb_sqlStmtColumns_!=null) numOfFields=pxb_sqlStmtColumns_.size();

for (int i = 0; i < numOfFields; i++) {

String columnName = (String) pxb_sqlStmtColumns_.get(i);

// fix the name quoting for px processing
if(!(tableInfo_ instanceof SQLVirtualTableInfo)){
columnName =
DatabaseTypeUtils.fullyQualifyColumnName(columnName, databaseType);
}

String alias = (String) pxb_sqlStmtAliasColumns_.get(i);

if ((alias != null) && (alias.length() > 0)) {
// fix the alias quoting for px processing
alias = DatabaseTypeUtils.fullyQualifyColumnName(alias,
databaseType);
}
else alias = null;
selectStatement.getSelectedFields().add(new
SelectedField(columnName, alias));
}

// append the virtual columns expressed by a SQL expression

ColumnInfo col = null;
if (pxb_selectedSQLExpressions_!=null)
{
for (Iterator sqlExpressions =
pxb_selectedSQLExpressions_.iterator(); sqlExpressions.hasNext();) {
col = (ColumnInfo) sqlExpressions.next();
String alias = col.getColumnAlias();
if (alias==null) alias = col.getColumnName();
selectStatement.getSelectedFields().add(new
SelectedField(col.getSQLExpression(),
DatabaseTypeUtils.fullyQualifyColumnName(alias, databaseType)));
}
}

pxb_tableName_ =
DatabaseTypeUtils.fullyQualifySchemaName(pxb_tableName_, databaseType);
selectStatement.setSelectedTable(pxb_tableName_);
if ((pxb_whereClause_ != null) && (pxb_whereClause_.length() > 0)) {
selectStatement.setWhereClause(pxb_whereClause_);
}

return selectStatement;
}

protected void generateInsertStatement() {

 String insertStmt = "insert into " + pxb_tableName_ + " values(\n";

int numOfFields = pxb_sqlStmtColumns_.size();

for (int i = 0; i < numOfFields; i++) {

// see if the column name was aliased to remove wild characters

String colName = (String) pxb_sqlStmtColumns_.get(i);
String alias = (String) pxb_sqlStmtAliasColumns_.get(i);
if ((alias != null) && (alias.length() > 0))
colName = alias;

insertStmt += ("ORCHESTRATE." + colName);

if (i < (numOfFields - 1)) {
insertStmt += ", ";
insertStmt += "\n";
} else {
insertStmt += ")";
}
}

pxb_insertStatement_ = insertStmt;
}

protected void generateUpdateStatement() {

if (pxb_updateStatement_ != null) {
return;
}

pxb_updateStatement_ = "";
}

protected void generateCreateStatement() {

generateCreateStatement(pxb_sqlStmtColumns_);
}

protected void generateCreateStatement(List sqlStmtColumns) {

DatabaseTypeSourceEnum databaseType = getDatabaseType();

boolean bFailOnError = (this.tableAction == TableAction.replace);
pxb_createStatementFailOnError_ = (bFailOnError) ? "1" : "0"; //TODO
expose for outside use

StringBuffer createTableStmt = new StringBuffer();

createTableStmt.append( "create table
" ).append( pxb_tableName_ ).append( " (\n" );

int numOfFields = sqlStmtColumns.size();

int i;
String dataType;
String columnName;

for (i = 0; i < numOfFields; i++) {

dataType = (String) pxb_columnDataTypes_.get(i);

columnName = (String) (sqlStmtColumns.get(i));

createTableStmt.append( columnName ).append( "

" ).append( dataType );
 /**
* If we are on SQL Server, make sure we use a Case sensitive
* collation sequence for all strings In Validate Analysis
Database,
* verify that a nvarchar column is case sensitive.
*/
if (DatabaseTypeSourceEnum.MSSQL_SERVER.equals(databaseType)) {
if (dataType.startsWith("nvarchar")) {
createTableStmt.append( " COLLATE
SQL_Latin1_General_CP1_CS_AS " );
}
}

/** Figure out whether to separate or not * */

if (i < (numOfFields - 1)) {
createTableStmt.append( ", " );
} else {
createTableStmt.append( ")" );
}

createTableStmt.append( "\n" );
}

// If compression enabled
if( this.pxb_compressionEnabled )
{
createTableStmt.append(" COMPRESS YES \n");
}

if (this.tableSpace!=null &&
DatabaseTypeSourceEnum.DB2.equals(databaseType)) {
createTableStmt.append( " in
" ).append(this.tableSpace).append(generatePartitionStatement()).append( " \n" );

pxb_createStatement_ = createTableStmt.toString() +
generateCreateStatementPredecate();

protected String generateCreateStatementPredecate()

{
StringBuffer predicate = new StringBuffer();
if(tableInfo_.getPartitionColumn() != null
&& DatabaseTypeSourceEnum.HIVE.equals(getDatabaseType())) {

predicate.append(" clustered
by(").append(tableInfo_.getPartitionColumn()).append(") into 50 buckets");
predicate.append(" stored as orc
tblproperties(\"transactional\"=\"true\")");
}
return predicate.toString();
}

protected String generatePartitionStatement()

{
return "";
 }

protected void generateBeforeAfterSQL()

{
}

private void addSourceProperties() {

//TODO: add below variables per specific Connector, as they are not
universal in context
// properties.put("TableName", pxb_tableName_);

String connectorName = getConnectorName();

properties.put("GenerateSQL", pxb_generateSql_);
if("NetezzaConnector".equals(connectorName))
properties.put("TypeMismatchActionSource", "0");
// Refer the following link for the meaning of "EnableQuotedIDs". This
has special meaning with JDBC.
// Other connecters use this property only in case of DDL/DML, but JDBC
uses this in all cases.
//
https://www.ibm.com/support/knowledgecenter/SSZJPZ_11.3.0/com.ibm.swg.im.iis.conn.j
dbc.usage.doc/topics/jdbc_enable_quoted_identifiers.html
if(connectorName.contains("JDBCConnector"))
{
properties.put("EnableQuotedIDs", "0");
}
else
{
properties.put("EnableQuotedIDs", "1");
}
properties.put("SelectStatement", pxb_selectStatement_);
properties.put("RecordCount", pxb_recordCount_);

properties.put("EndOfWave", pxb_EndOfWave_);

properties.put("IsolationLevel", pxb_isolationLevel_);
properties.put("AutocommitMode", pxb_autoCommit_);
properties.put("ArraySize", pxb_arraySize_);
if(connectorName.contains("HiveConnector")) {
properties.put("BatchSize", pxb_batchSize_);
}
properties.put("PrefixForExpressionColumns","EXPR");
properties.put("FailOnSizeMismatch", pxb_ignoreSizeChecks_);
properties.put("FailOnTypeMismatch", pxb_ignoreTypeChecks_);
properties.put("PassLobLocator", pxb_passLob_);
// customProperties_.put("Column", "");
properties.put("CodePage", pxb_codePage_);
if ( "DB2Connector".equals(connectorName) && jobProperties != null &&
jobProperties.getJobOptions() != null)
{
properties.put("KeepConductorConnectionAlive", (new
Boolean(jobProperties.getJobOptions().keepConductorConnectionAlive())).booleanValue
() ? "1" : "0");
}

if(DatabaseTypeSourceEnum.HIVE.equals(getDatabaseType()))
{
if (jobProperties != null
&& jobProperties.getJobOptions() != null
 &&
jobProperties.getJobOptions().getHiveProperties() != null)
{
pxb_beforeStatement =
jobProperties.getJobOptions().getHiveProperties().trim();
if(pxb_beforeStatement.length() > 0)
{
properties.put("BeforeAfter", "1");
properties.put("BeforeSQL", pxb_beforeStatement);
properties.put("BeforeSQL/FailOnError", "0");
}
}
}
}

private void addTargetProperties() {

//TODO: add below variables per specific Connector, as they are not
universal in context
String connectorName = getConnectorName();
DatabaseTypeSourceEnum databaseType = getDatabaseType();

properties.put("WriteMode", String.valueOf(this.writeMode.getValue()));
properties.put("GenerateSQL", pxb_generateSql_);
String tableName = pxb_tableName_;

properties.put("TableName", tableName);
properties.put("EnableQuotedIDs", "0");
properties.put("TableAction",
String.valueOf(this.tableAction.getValue()));
properties.put("GenerateCreateStatement",
pxb_generateCreateStatement_);
properties.put("GenerateCreateStatement/FailOnError",
pxb_createStatementFailOnError_);
properties.put("CreateStatement", pxb_createStatement_);
properties.put("GenerateDropStatement", "1");
properties.put("GenerateDropStatement/FailOnError", "0");
properties.put("InsertStatement", pxb_insertStatement_);
properties.put("UpdateStatement", pxb_updateStatement_);
properties.put("RecordCount", pxb_recordCount_);
properties.put("IsolationLevel", pxb_isolationLevel_);
properties.put("AutocommitMode", pxb_autoCommit_);
properties.put("ArraySize", pxb_arraySize_);
if(connectorName.contains("HiveConnector")) {
properties.put("BatchSize", pxb_batchSize_);
}
properties.put("FailOnSizeMismatch", pxb_ignoreSizeChecks_);
properties.put("FailOnTypeMismatch", pxb_ignoreTypeChecks_);
properties.put("DropUnmatchedFields", "1");
if(DatabaseTypeSourceEnum.HIVE.equals(databaseType)) {
properties.put("RecordOrdering", "0");
} else {
properties.put("RecordOrdering", "1");
}
properties.put("EndOfWave", pxb_EndOfWave_);
if (pxb_beforeStatement!=null || pxb_afterStatement!=null)
{
properties.put("BeforeAfter", "1");
} else
 {
properties.put("BeforeAfter", "0");
}
if (pxb_beforeStatement!=null)
{
properties.put("before", pxb_beforeStatement);
properties.put("before/FailOnError", "0");
}
if (pxb_afterStatement!=null)
{
properties.put("after", pxb_afterStatement);
properties.put("after/FailOnError", "0");
}

if (writeMode==WriteMode.bulk_load)
{
properties.put("LoadControl/MessageFile", "loadMsgs.out");
}
}

private void setFieldInfo(TableInfo tableInfo) {

// Get field information. First check to see if
// column info is in TableInfo object. If no column
// info in TableInfo object, then check xml string.
// If no field info in xml string, fail.
List columnList = tableInfo.getPerColumnData();
int columnCnt = columnList.size();
if (columnCnt > 0) // Column info is stored in ColumnInfo objects
{
pxb_sqlStmtColumns_ = new ArrayList(columnCnt);
pxb_sqlStmtAliasColumns_ = new ArrayList(columnCnt);
pxb_nullableColumns_ = new ArrayList(columnCnt);
pxb_columnPXTypes_ = new ArrayList(columnCnt);
pxb_columnDataTypes_ = new ArrayList(columnCnt);
pxb_columnNativeTypes_ = new ArrayList(columnCnt);
pxb_selectedSQLExpressions_ = new ArrayList(columnCnt);

ColumnInfo columnInfo = null;

String columnName = null;
String alias1 = null;
String alias2 = null;
for (int i = 0; i < columnCnt; i++) {

columnInfo = (ColumnInfo) columnList.get(i);

// if the column is a virtual column defined by a SQL
// expression, put it in a different list
if (columnInfo.getSQLExpression() != null &&
columnInfo.getSQLExpression().length() > 0) {
pxb_selectedSQLExpressions_.add(columnInfo);
continue;
}

columnName = columnInfo.getColumnName();

// Determine if virtual column, or regular column

if (columnInfo.getIsVirtualColumn()) {
setVirtualColumnFieldInfo(columnInfo);
} else {
// check if the same column with the same alias
already
// exists
boolean colAlreadyExists = false;
for (int j = 0; j < pxb_sqlStmtColumns_.size(); j++)
{
if
(columnName.equals(pxb_sqlStmtColumns_.get(j))) {
alias1 = columnInfo.getColumnAlias();
if (alias1 == null)
alias1 = "";
alias2 = (String)
pxb_sqlStmtAliasColumns_.get(j);
if (alias2 == null)
alias2 = "";
if (alias1.equals(alias2)) {
colAlreadyExists = true;
break;
}
}

}
if (!colAlreadyExists) {
String alias = columnInfo.getColumnAlias();

if(DatabaseTypeSourceEnum.HIVE.equals(getDatabaseType()))
{
pxb_sqlStmtColumns_.add(columnName==null?
columnName:columnName.toLowerCase());
pxb_sqlStmtAliasColumns_.add(alias==null?
alias:alias.toLowerCase());
}
else
{
pxb_sqlStmtColumns_.add(columnName);
pxb_sqlStmtAliasColumns_.add(alias);
}

pxb_nullableColumns_.add(columnInfo.getIsNullable() ? "N" : "");

pxb_columnPXTypes_.add(columnInfo.getPxMetaFormat());

pxb_columnDataTypes_.add(columnInfo.getDataType());

pxb_columnNativeTypes_.add(columnInfo.getNativeType());
}
}
}

}
}

private void setVirtualColumnFieldInfo(ColumnInfo columnInfo) {

List columnInfoList = columnInfo.getVirtualColumnBaseColumns();

// If virtual column, need to get the base columns info.

int numOfBaseColumns = columnInfoList.size();
int i;

ColumnInfo baseColumnInfo = null;

 String baseColumnName = null;
String alias1 = null;
String alias2 = null;
for (i = 0; i < numOfBaseColumns; i++) {
baseColumnInfo = (ColumnInfo) columnInfoList.get(i);
baseColumnName = baseColumnInfo.getColumnName();

// Don't want to add duplicate columns to connection

// select statement.
boolean columnAlreadyExists = false;
if ((pxb_sqlStmtColumns_.contains(baseColumnName))) {
for (int j = 0; j < pxb_sqlStmtColumns_.size(); j++) {
if
(baseColumnName.equals(pxb_sqlStmtColumns_.get(j))) {
alias1 = baseColumnInfo.getColumnAlias();
if (alias1 == null)
alias1 = "";
alias2 = (String)
pxb_sqlStmtAliasColumns_.get(j);
if (alias2 == null)
alias2 = "";
if (alias1.equals(alias2)) {
columnAlreadyExists = true;
break;
}
}
}
}
if (!columnAlreadyExists) {
pxb_sqlStmtColumns_.add(baseColumnName);

pxb_sqlStmtAliasColumns_.add(baseColumnInfo.getColumnAlias());
pxb_nullableColumns_.add(baseColumnInfo.getIsNullable() ?
"N" : "");
pxb_columnPXTypes_.add(baseColumnInfo.getPxMetaFormat());
pxb_columnDataTypes_.add(baseColumnInfo.getDataType());
pxb_columnNativeTypes_.add(baseColumnInfo.getNativeType());
}
}

//###########################################################################
###
// Some helper classes to help building the select statement in a parsed form
//###########################################################################
###

public static class SelectStatement

{
private List<SelectedField> selectedFields = new
ArrayList<SelectedField>();
private String selectedTable;
private String whereClause;

public SelectStatement()
{
}
 public List<SelectedField> getSelectedFields()
{
return selectedFields;
}

public String getSelectedTable()

{
return selectedTable;
}

public void setSelectedTable(String table)

{
this.selectedTable = table;
}

public String getWhereClause()

{
return whereClause;
}

public void setWhereClause(String whereClause)

{
this.whereClause = whereClause;
}
}

public static class SelectedField

{
private String expression;
private String alias;

public SelectedField(String expression, String alias)

{
this.expression = expression;
this.alias = alias;
}

public String getExpression()

{
return expression;
}

public String getAlias()

{
return alias;
}

public String getSelectName()

{
if (alias!=null) return alias;
else return expression;
}
}