irext/core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[undone] removed Ali-OSS related logic from irext console

Browse Source
This commit is contained in:
strawmanbobi
2016-12-04 12:14:03 +08:00
parent 067f428f6f
commit 2ffa59e204
1 changed files with 224 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
src/web_console/work_unit/internal_logic.js
View File

@@ -165,19 +165,9 @@ exports.downloadRemoteBinCachedWorkUnit = function(remoteIndexID, callback) {
logger.info("file " + localBinFileName + " already exists, serve directly"); logger.info("file " + localBinFileName + " already exists, serve directly");
callback(error, localBinFileName); callback(error, localBinFileName);
} else { } else {
logger.info("file " + localBinFileName + " does not exist, download it from OSS"); logger.info("file " + localBinFileName + " does not exist");
var aliOss = new OSS(OSS_HOST, DEBUG_BUCKET_NAME, OSS_APP_ID, OSS_APP_SECRET);
aliOss.serveObjectByID(fileName, localBinFileName,
function (serveObjectErr, response) {
if (errorCode.SUCCESS.code == serveObjectErr) {
logger.info("serve remote binary object and cached successfully");
callback(error, localBinFileName);
} else {
logger.info("serve remote binary object and cached failed");
error = errorCode.FAILED; error = errorCode.FAILED;
callback(error, null); callback(error, NULL);
}
});
} }
}); });
} else { } else {
@@ -290,7 +280,8 @@ exports.createRemoteIndexWorkUnit = function(remoteIndex, filePath, contentType,
// step 3, if successfully created tag binary file, upload binary to OSS // step 3, if successfully created tag binary file, upload binary to OSS
logger.info("remote " + remoteIndex.remote_name + " has successfully been generated"); logger.info("remote " + remoteIndex.remote_name + " has successfully been generated");
outputPath = fileDir; outputPath = fileDir;
newACRemoteNumber = remoteIndex.remote_name.substring(remoteIndex.remote_name.lastIndexOf('_') + 1); newACRemoteNumber = remoteIndex.remote_name
.substring(remoteIndex.remote_name.lastIndexOf('_') + 1);
outputFilePath = outputPath + "/irda_" + tagType + "_" + newACRemoteNumber + ".bin"; outputFilePath = outputPath + "/irda_" + tagType + "_" + newACRemoteNumber + ".bin";
logger.info("final tag binary output file = " + outputFilePath); logger.info("final tag binary output file = " + outputFilePath);
@@ -302,7 +293,8 @@ exports.createRemoteIndexWorkUnit = function(remoteIndex, filePath, contentType,
logger.info("read remote binary file successfully, file size = " + fileData.length); logger.info("read remote binary file successfully, file size = " + fileData.length);
////////////////////////////////////// //////////////////////////////////////
// step 3.5, check if this remote index is already contained in remote index list by binary // step 3.5, check if this remote index is already
// contained in remote index list by binary
var fileHash = checksum(fileData); var fileHash = checksum(fileData);
logger.info("hash of binary file = " + fileHash); logger.info("hash of binary file = " + fileHash);
var conditions = { var conditions = {
@@ -318,14 +310,6 @@ exports.createRemoteIndexWorkUnit = function(remoteIndex, filePath, contentType,
logger.info("this remote is duplicated by binary value"); logger.info("this remote is duplicated by binary value");
callback(errorCode.DUPLICATED_REMOTE_CODE, null); callback(errorCode.DUPLICATED_REMOTE_CODE, null);
} else { } else {
var aliOss = new OSS(OSS_HOST, DEBUG_BUCKET_NAME, OSS_APP_ID, OSS_APP_SECRET);
// do not specify objectID for newly created media
outputFileName = "irda_" + tagType + "_" + newACRemoteNumber + ".bin";
aliOss.saveObjectFromBinary(outputFileName, fileData, contentType,
function (createObjectErr, objectID) {
if (errorCode.SUCCESS.code == createObjectErr) {
// deprecated: do not delete the output binary file for future use
//////////////////////////////////////
// step 4, create remote index record in db // step 4, create remote index record in db
remoteIndex.remote_name = newACRemoteNumber + ""; remoteIndex.remote_name = newACRemoteNumber + "";
@@ -360,14 +344,8 @@ exports.createRemoteIndexWorkUnit = function(remoteIndex, filePath, contentType,
if(errorCode.SUCCESS.code == findRemoteIndexErr.code && if(errorCode.SUCCESS.code == findRemoteIndexErr.code &&
remoteIndexes && remoteIndexes &&
remoteIndexes.length > 0) { remoteIndexes.length > 0) {
logger.info("remote index already exists, failed this time"); logger.info("remote index already exists");
/*
RemoteIndex.updateRemoteIndex(remoteIndexes[0].id,
newRemoteIndex,
function(updateRemoteIndexErr, updatedRemoteIndex) {
callback(updateRemoteIndexErr, updatedRemoteIndex);
});
*/
callback(errorCode.FAILED, null); callback(errorCode.FAILED, null);
} else { } else {
RemoteIndex.createRemoteIndex(newRemoteIndex, RemoteIndex.createRemoteIndex(newRemoteIndex,
@@ -376,11 +354,6 @@ exports.createRemoteIndexWorkUnit = function(remoteIndex, filePath, contentType,
}); });
} }
}); });
} else {
logger.error("save file object to OSS failed : " + createObjectErr);
callback(errorCode.FAILED, null);
}
});
} }
}); });
} }
@@ -401,10 +374,6 @@ exports.createRemoteIndexWorkUnit = function(remoteIndex, filePath, contentType,
protocolPath = fileDir + "/" + PROTOCOL_PATH + "/"; protocolPath = fileDir + "/" + PROTOCOL_PATH + "/";
protocolFileName = remoteIndex.protocol_name + ".bin"; protocolFileName = remoteIndex.protocol_name + ".bin";
localProtocolFileName = protocolPath + remoteIndex.protocol_name + ".bin"; localProtocolFileName = protocolPath + remoteIndex.protocol_name + ".bin";
var aliOss = new OSS(OSS_HOST, PROTOCOL_BUCKET_NAME, OSS_APP_ID, OSS_APP_SECRET);
aliOss.serveObjectByID(protocolFileName, localProtocolFileName,
function (serveObjectErr, response) {
if (errorCode.SUCCESS.code == serveObjectErr) {
logger.info("protocol binary fetched from OSS, continue processing with remote file"); logger.info("protocol binary fetched from OSS, continue processing with remote file");
remoteXMLFilePath = fileDir + "/" + remoteIndex.remote_name + ".xml"; remoteXMLFilePath = fileDir + "/" + remoteIndex.remote_name + ".xml";
logger.info("remote XML file path = " + remoteXMLFilePath); logger.info("remote XML file path = " + remoteXMLFilePath);
@@ -434,18 +403,17 @@ exports.createRemoteIndexWorkUnit = function(remoteIndex, filePath, contentType,
remoteBinFilePath = fileDir + "/" + remoteDir + remoteIndex.protocol_name + "#" + remoteBinFilePath = fileDir + "/" + remoteDir + remoteIndex.protocol_name + "#" +
remoteIndex.remote_name + ".bin"; remoteIndex.remote_name + ".bin";
userArgs.length = 0; userArgs.length = 0;
// python irda_tv_merge.py [protocol_dir_abs_path] [remote_bin_file_abs_path] [output_$category_dir_abs_path] // python irda_tv_merge.py [protocol_dir_abs_path]
// [remote_bin_file_abs_path] [output_$category_dir_abs_path]
logger.info("protocol path = " + protocolPath + ", remote bin path = " + remoteBinFilePath + logger.info("protocol path = " + protocolPath + ", remote bin path = " + remoteBinFilePath +
", output = " + outputPath); ", output = " + outputPath);
userArgs.push(protocolPath); userArgs.push(protocolPath);
userArgs.push(remoteBinFilePath); userArgs.push(remoteBinFilePath);
userArgs.push(outputPath); userArgs.push(outputPath);
pythonCaller.call(pythonRuntimeDir, pythonFile, userArgs, function(remoteMergeErr, mergeResult) { pythonCaller.call(pythonRuntimeDir, pythonFile, userArgs,
function(remoteMergeErr, mergeResult) {
logger.info("merge protocol error = " + remoteMergeErr); logger.info("merge protocol error = " + remoteMergeErr);
if(errorCode.SUCCESS.code == remoteMergeErr) { if(errorCode.SUCCESS.code == remoteMergeErr) {
//////////////////////////////////////
// step 5, upload file to aliyun OSS
outputFilePath = outputPath + "/irda_" + remoteIndex.protocol_name + "_" + outputFilePath = outputPath + "/irda_" + remoteIndex.protocol_name + "_" +
remoteIndex.remote_name + ".bin"; remoteIndex.remote_name + ".bin";
logger.info("final output file = " + outputFilePath); logger.info("final output file = " + outputFilePath);
@@ -455,10 +423,11 @@ exports.createRemoteIndexWorkUnit = function(remoteIndex, filePath, contentType,
logger.error("read remote code binary file error : " + readFileErr); logger.error("read remote code binary file error : " + readFileErr);
callback(errorCode.FAILED, null); callback(errorCode.FAILED, null);
} else { } else {
logger.info("read remote binary file successfully, file size = " + fileData.length); logger.info("read remote binary file successfully, " +
"file size = " + fileData.length);
////////////////////////////////////// //////////////////////////////////////
// step 5.5, check if this remote index is already contained in remote index list by binary // step 5, check if this remote index is already contained in remote index
// list by binary
var fileHash = checksum(fileData); var fileHash = checksum(fileData);
logger.info("hash of binary file = " + fileHash); logger.info("hash of binary file = " + fileHash);
var conditions = null; var conditions = null;
@@ -484,15 +453,6 @@ exports.createRemoteIndexWorkUnit = function(remoteIndex, filePath, contentType,
logger.info("this remote is duplicated by binary value"); logger.info("this remote is duplicated by binary value");
callback(errorCode.DUPLICATED_REMOTE_CODE, null); callback(errorCode.DUPLICATED_REMOTE_CODE, null);
} else { } else {
// do not specify objectID for newly created media
outputFileName = "irda_" + remoteIndex.protocol_name + "_" +
remoteIndex.remote_name + ".bin";
aliOss = new OSS(OSS_HOST, DEBUG_BUCKET_NAME, OSS_APP_ID, OSS_APP_SECRET);
aliOss.saveObjectFromBinary(outputFileName, fileData, contentType,
function (createObjectErr, objectID) {
if (errorCode.SUCCESS.code == createObjectErr) {
// deprecated: do not delete the output binary file for future use
////////////////////////////////////// //////////////////////////////////////
// step 6, create remote index record in db // step 6, create remote index record in db
if (remoteIndex.category_id == enums.CATEGORY_STB) { if (remoteIndex.category_id == enums.CATEGORY_STB) {
@@ -549,13 +509,7 @@ exports.createRemoteIndexWorkUnit = function(remoteIndex, filePath, contentType,
if(errorCode.SUCCESS.code == findRemoteIndexErr.code && if(errorCode.SUCCESS.code == findRemoteIndexErr.code &&
remoteIndexes && remoteIndexes &&
remoteIndexes.length > 0) { remoteIndexes.length > 0) {
logger.info("remote index already exists, failed this time"); logger.info("remote index already exists");
/*
RemoteIndex.updateRemoteIndex(remoteIndexes[0].id, newRemoteIndex,
function(updateRemoteIndexErr, updatedRemoteIndex) {
callback(updateRemoteIndexErr, updatedRemoteIndex);
});
*/
callback(errorCode.FAILED, null); callback(errorCode.FAILED, null);
} else { } else {
RemoteIndex.createRemoteIndex(newRemoteIndex, RemoteIndex.createRemoteIndex(newRemoteIndex,
@@ -564,11 +518,6 @@ exports.createRemoteIndexWorkUnit = function(remoteIndex, filePath, contentType,
}); });
} }
}); });
} else {
logger.error("save file object to OSS failed : " + createObjectErr);
callback(errorCode.FAILED, null);
}
});
} }
}); });
} }
@@ -585,11 +534,6 @@ exports.createRemoteIndexWorkUnit = function(remoteIndex, filePath, contentType,
logger.error('failed to execute python script from application'); logger.error('failed to execute python script from application');
callback(errorCode.FAILED, null); callback(errorCode.FAILED, null);
} }
} else {
logger.info("failed to fetch protocol binary file, return with failure");
callback(errorCode.FAILED, null);
}
});
} }
} else { } else {
logger.info("invalid admin ID, return directly"); logger.info("invalid admin ID, return directly");
@@ -614,9 +558,13 @@ exports.deleteRemoteIndexWorkUnit = function (remoteIndex, adminID, callback) {
} }
} }
var requestSender = var requestSender =
new RequestSender(PRIMARY_SERVER_ADDRESS, PRIMARY_SERVER_PORT, DELETE_REMOTE_INDEX_SERVICE, queryParams); new RequestSender(PRIMARY_SERVER_ADDRESS,
PRIMARY_SERVER_PORT,
DELETE_REMOTE_INDEX_SERVICE,
queryParams);
requestSender.sendPostRequest(remoteIndex, function(deleteRemoteIndexesRequestErr, deleteRemoteIndexesResponse) { requestSender.sendPostRequest(remoteIndex,
function(deleteRemoteIndexesRequestErr, deleteRemoteIndexesResponse) {
/* /*
if(errorCode.SUCCESS.code == deleteRemoteIndexesRequestErr && if(errorCode.SUCCESS.code == deleteRemoteIndexesRequestErr &&
JSON.parse(deleteRemoteIndexesResponse).status.code == errorCode.SUCCESS.code) { JSON.parse(deleteRemoteIndexesResponse).status.code == errorCode.SUCCESS.code) {
@@ -689,8 +637,8 @@ exports.publishRemoteIndexWorkUnit = function (callback) {
var fileDir = unixFilePath.substring(0, lios); var fileDir = unixFilePath.substring(0, lios);
var outputFileName = ''; var outputFileName = '';
var uploadedRIIds = []; var uploadedRIIds = [];
var downloadOssFilePath = ''; var sourceFileName = '';
var localFilePath = ''; var targetFileName = '';
var conditions = null; var conditions = null;
conditions = { conditions = {
@@ -705,58 +653,28 @@ exports.publishRemoteIndexWorkUnit = function (callback) {
if(errorCode.SUCCESS.code == findRemoteIndexErr.code) { if(errorCode.SUCCESS.code == findRemoteIndexErr.code) {
logger.info("find remote indexes successfully, size of remote index list : " + remoteIndexes.length); logger.info("find remote indexes successfully, size of remote index list : " + remoteIndexes.length);
////////////////////////////////////// //////////////////////////////////////
// step 2, download binaries from debug bucket and upload them to release bucket
var aliOss = new OSS(OSS_HOST, DEBUG_BUCKET_NAME, OSS_APP_ID, OSS_APP_SECRET);
async.eachSeries(remoteIndexes, function (remoteIndex, innerCallback) { async.eachSeries(remoteIndexes, function (remoteIndex, innerCallback) {
var remoteName = remoteIndex.remote; var remoteName = remoteIndex.remote;
var protocolName = remoteIndex.protocol; var protocolName = remoteIndex.protocol;
var binFileName = fileDir + "/irda_" + protocolName + "_" + remoteName + ".bin"; var binFileName = fileDir + "/irda_" + protocolName + "_" + remoteName + ".bin";
logger.info("binary file name = " + binFileName); logger.info("binary file name = " + binFileName);
// step 2, copy file to transfer path
downloadOssFilePath = "irda_" + protocolName + "_" + sourceFileName = FILE_TEMP_PATH + "/irda_" + protocolName + "_" +
remoteName + ".bin"; remoteName + ".bin";
localFilePath = FILE_TEMP_PATH + "/binary_transfer/" + downloadOssFilePath; targetFileName = FILE_TEMP_PATH + "/binary_transfer" + "/irda_" + protocolName + "_" +
logger.info("local file path = " + localFilePath); remoteName + ".bin";
aliOss.serveObjectByID(downloadOssFilePath, localFilePath, var readStream = fs.createReadStream(sourceFileName);
function (serveObjectErr, response) { var writeStream = fs.createWriteStream(targetFileName);
if (errorCode.SUCCESS.code == serveObjectErr) { readStream.pipe(writeStream);
logger.info("serve remote binary object successfully : " + downloadOssFilePath); logger.info("copy remote binary object successfully : " + targetFileName);
fs.readFile(targetFileName, function(readFileErr, fileData) {
fs.readFile(localFilePath, function(readFileErr, fileData) {
if (readFileErr) { if (readFileErr) {
logger.error("read remote code binary file error : " + readFileErr); logger.error("read remote code binary file error : " + readFileErr);
innerCallback(); innerCallback();
} else { } else {
logger.info("read remote binary file successfully, file size = " + fileData.length); logger.info("read remote binary file successfully, file size = " + fileData.length);
// TODO: send HTTP post request to release server
// do not specify objectID for newly created media
outputFileName = "irda_" + protocolName + "_" +
remoteName + ".bin";
aliOss = new OSS(OSS_HOST, RELEASE_BUCKET_NAME, OSS_APP_ID, OSS_APP_SECRET);
aliOss.saveObjectFromBinary(outputFileName, fileData, "application/octet-stream",
function (createObjectErr, objectID) {
if (errorCode.SUCCESS.code == createObjectErr) {
logger.info("successfully uploaded file " + binFileName);
// do not update status for unpublished remote indexes here since we need do more things
/*
RemoteIndex.publishRemoteIndex(remoteIndex.id, enums.ITEM_VALID, function(publishRemoteIndexErr) {
// process next anyway
innerCallback();
});
*/
uploadedRIIds.push(remoteIndex.id);
innerCallback();
} else {
logger.error("upload file " + binFileName + " failed");
innerCallback();
}
});
}
});
} else {
logger.error("failed to download remote binary from bucket");
innerCallback();
} }
}); });
}, function(err) { }, function(err) {
@@ -784,7 +702,10 @@ exports.publishRemoteIndexWorkUnit = function (callback) {
queryParams.put("app_token", REQUEST_APP_TOKEN); queryParams.put("app_token", REQUEST_APP_TOKEN);
var requestSender = var requestSender =
new RequestSender(PRIMARY_SERVER_ADDRESS, PRIMARY_SERVER_PORT, PUBLISH_REMOTE_INDEX_SERVICE, queryParams); new RequestSender(PRIMARY_SERVER_ADDRESS,
PRIMARY_SERVER_PORT,
PUBLISH_REMOTE_INDEX_SERVICE,
queryParams);
requestSender.sendPostRequest(remoteIndexes, requestSender.sendPostRequest(remoteIndexes,
function(publishRemoteIndexesRequestErr, publishRemoteIndexesResponse) { function(publishRemoteIndexesRequestErr, publishRemoteIndexesResponse) {