#include "Utils/leak_detector.h" #pragma comment(linker, "/STACK:536870912") // 512 МБ #include #include #include #include #include #include #include #include #include #include #include #if _WIN32 && _DEBUG #include #endif #define DEBUG_LVL1 true #define RELEASE_CANDIDATE 0 #include "ParallelizationRegions/ParRegions_func.h" #include "ParallelizationRegions/resolve_par_reg_conflicts.h" #include "ParallelizationRegions/expand_extract_reg.h" #include "Transformations/replace_dist_arrays_in_io.h" #include "Distribution/Distribution.h" #include "Distribution/GraphCSR.h" #include "Distribution/Arrays.h" #include "Distribution/DvmhDirective.h" #include "CreateInterTree/CreateInterTree.h" #include "Utils/errors.h" #include "Utils/SgUtils.h" #include "Utils/module_utils.h" #include "ProjectManipulation/ParseFiles.h" #include "ProjectManipulation/PerfAnalyzer.h" #include "ProjectManipulation/ConvertFiles.h" #include "LoopAnalyzer/loop_analyzer.h" #include "GraphCall/graph_calls_func.h" #include "GraphLoop/graph_loops_func.h" #include "DynamicAnalysis/gCov_parser_func.h" #include "DynamicAnalysis/createParallelRegions.h" #include "DirectiveProcessing/directive_analyzer.h" #include "DirectiveProcessing/directive_creator.h" #include "DirectiveProcessing/insert_directive.h" #include "DirectiveProcessing/directive_omp_parser.h" #include "VerificationCode/verifications.h" #include "Distribution/CreateDistributionDirs.h" #include "PrivateAnalyzer/private_analyzer.h" #include "ExpressionTransform/expr_transform.h" #include "Predictor/PredictScheme.h" #include "Predictor/PredictorModel.h" #include "SageAnalysisTool/depInterfaceExt.h" #include "DvmhRegions/DvmhRegionInserter.h" #include "DvmhRegions/LoopChecker.h" #include "DvmhRegions/ReadWriteAnalyzer.h" #include "Utils/utils.h" #include "Distribution/Array.h" #include "VisualizerCalls/get_information.h" #include "VisualizerCalls/SendMessage.h" #include "VisualizerCalls/BuildGraph.h" #include "Transformations/enddo_loop_converter.h" #include "Transformations/loop_transform.h" #include "Transformations/array_assign_to_loop.h" #include "Transformations/private_arrays_resizing.h" #include "Transformations/loops_splitter.h" #include "Transformations/loops_combiner.h" #include "Transformations/loops_unrolling.h" #include "Transformations/uniq_call_chain_dup.h" #include "Transformations/checkpoints.h" #include "Transformations/swap_array_dims.h" #include "Transformations/function_purifying.h" #include "Transformations/private_removing.h" #include "Transformations/fix_common_blocks.h" #include "Transformations/convert_to_c.h" #include "Transformations/set_implicit_none.h" #include "Transformations/dead_code.h" #include "RenameSymbols/rename_symbols.h" #include "ProjectParameters/projectParameters.h" #include "CFGraph/IR.h" #include "CFGraph/RD_subst.h" #include "CFGraph/CFGraph.h" #include "CFGraph/live_variable_analysis.h" #include "CFGraph/private_variables_analysis.h" #include "Inliner/inliner.h" #include "dvm.h" #include "Sapfor.h" #include "Utils/PassManager.h" using namespace std; using std::chrono::high_resolution_clock; using std::chrono::duration_cast; using std::chrono::milliseconds; int PASSES_DONE[EMPTY_PASS]; bool PASSES_DONE_INIT = false; int *ALGORITHMS_DONE[EMPTY_ALGO] = { NULL }; #include "SapforData.h" static SgProject *project = NULL; // for pass temporary functions from DEF_USE_STAGE1 to SUBST_EXPR static map> temporaryAllFuncInfo = map>(); //extern void runRenameSymbolsByFiles(SgFile* file, SgProject* proj); //extern void runRenameSymbols(SgProject* proj); static auto globalTime = high_resolution_clock::now(); void deleteAllAllocatedData(bool enable) { if (enable) { for (auto& toDel : declaredArrays) { if (!toDel.second.first->IsTemplate()) { delete toDel.second.first; delete toDel.second.second; } } declaredArrays.clear(); for (int i = 0; i < parallelRegions.size(); ++i) delete parallelRegions[i]; parallelRegions.clear(); for (int i = 0; i < subs_parallelRegions.size(); ++i) delete subs_parallelRegions[i]; subs_parallelRegions.clear(); shortFileNames.clear(); for (auto &it : allFuncInfo) for (auto &toDel : it.second) delete toDel; allFuncInfo.clear(); for (auto &it : allFuncInfo_IR) for (auto &toDel : it.second) delete toDel; allFuncInfo_IR.clear(); for (auto &elem : temporaryAllFuncInfo) for (auto &toDel : elem.second) delete toDel; temporaryAllFuncInfo.clear(); for (auto &loop : loopGraph) for (auto &toDel : loop.second) delete toDel; loopGraph.clear(); for (auto &toDel : depInfoForLoopGraph) delete toDel.second; depInfoForLoopGraph.clear(); commentsToInclude.clear(); SPF_messages.clear(); for (int i = 0; i < EMPTY_ALGO; ++i) delete[]ALGORITHMS_DONE[i]; delete project; GraphsKeeper::deleteGraphsKeeper(); deletePointerAllocatedData(); } } static inline void printDvmActiveDirsErrors() { for (auto &err : dvmDirErrors) { vector &currMessages = getObjectForFileFromMap(err.first.c_str(), SPF_messages); for (auto &code : err.second) { __spf_print(1, " ERROR: at line %d: Active DVM directives are not supported yet\n", code); currMessages.push_back(Messages(ERROR, code, R53, L"Active DVM directives are not supported (turn on DVM-directive support option)", 1020)); } } } extern "C" void printLowLevelWarnings(const char *fileName, const int line, const wchar_t* messageR, const char* messageE, const int group) { vector &currM = getObjectForFileFromMap(fileName, SPF_messages); __spf_print(1, "WARR: line %d: %s\n", line, messageE); if (!messageE) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); if (messageR) currM.push_back(Messages(WARR, line, messageR, to_wstring(messageE), group)); else currM.push_back(Messages(WARR, line, to_wstring(messageE), to_wstring(messageE), group)); } extern "C" void printLowLevelNote(const char *fileName, const int line, const wchar_t *messageR, const char *messageE, const int group) { vector &currM = getObjectForFileFromMap(fileName, SPF_messages); __spf_print(1, "NOTE: line %d: %s\n", line, messageE); currM.push_back(Messages(NOTE, line, messageR, to_wstring(messageE), group)); } static bool isDone(const int curr_regime) { if (PASSES_DONE[curr_regime] == 0) { if (curr_regime < EMPTY_PASS) PASSES_DONE[curr_regime] = 1; else printInternalError(convertFileName(__FILE__).c_str(), __LINE__); } // dont run passes twice else { if (curr_regime == PRIVATE_ANALYSIS_SPF) { if (PASSES_DONE[LOOP_ANALYZER_DATA_DIST_S1] == 1) return true; else if (passesIgnoreStateDone.find((passes)curr_regime) == passesIgnoreStateDone.end()) return true; } else { if (passesIgnoreStateDone.find((passes)curr_regime) == passesIgnoreStateDone.end()) return true; } } return false; } static void updateStatsExprs(const int id, const string &file) { auto node1 = current_file->firstStatement(); for (; node1; node1 = node1->lexNext()) sgStats[node1->thebif] = make_pair(file, id); auto node = current_file->firstStatement()->thebif; for (; node; node = node->thread) if (sgStats.find(node) == sgStats.end()) sgStats[node] = make_pair(file, id); for (SgExpression *ex = current_file->firstExpression(); ex; ex = ex->nextInExprTable()) sgExprs[ex->thellnd] = make_pair(file, id); } static map filesNameWithoutExt; static string unparseProjectIfNeed(SgFile* file, const int curr_regime, const bool need_to_unparse, const char* newVer, const char* folderName, set& allIncludeFiles, bool toString = false) { string unparseToBuf = ""; const char* file_name = file->filename(); if (curr_regime == CORRECT_CODE_STYLE || need_to_unparse && (folderName == NULL || (string(folderName) != "---modification---"))) { restoreCorrectedModuleProcNames(file); if (keepSpfDirs) revertion_spf_dirs(file, declaredArrays, declaratedArraysSt); else { vector toDel; for (SgStatement* st = file->firstStatement(); st; st = st->lexNext()) if (isSPF_stat(st)) // except sapfor parallel regions and if attributes dont move if (st->variant() != SPF_PARALLEL_REG_DIR && st->variant() != SPF_END_PARALLEL_REG_DIR) toDel.push_back(st); for (auto& elem : toDel) elem->deleteStmt(); } if (curr_regime == CORRECT_CODE_STYLE && newVer == NULL) newVer = ""; /*if (curr_regime == CORRECT_CODE_STYLE) groupDeclarations(file);*/ if (newVer == NULL) { __spf_print(1, " ERROR: null file addition name\n"); getObjectForFileFromMap(file_name, SPF_messages).push_back(Messages(ERROR, 1, R102, L"Internal error during unparsing process has occurred", 2007)); throw(-1); } string fout_name = ""; const string outExt = (out_free_form == 1) ? "f90" : "for"; string fPartOfName = OnlyName(file_name); if (!toString) if (filesNameWithoutExt.find(fPartOfName)->second != 1) fPartOfName = FullNameWithExt(file_name); if (folderName == NULL) fout_name = fPartOfName + "_" + newVer + "." + outExt; else { if (strlen(newVer) == 0) fout_name = folderName + string("/") + fPartOfName + "." + outExt; else fout_name = folderName + string("/") + fPartOfName + "_" + newVer + "." + outExt; } if (toString) __spf_print(1, " Unparsing to string\n"); else __spf_print(1, " Unparsing to <%s> file\n", fout_name.c_str()); if (curr_regime == INSERT_INCLUDES) { __spf_print(1, " try to find file <%s>\n", file_name); __spf_print(1, " in set %d, result %d\n", (int)filesToInclude.size(), filesToInclude.find(file_name) != filesToInclude.end()); } //TODO: add freeForm for each file if (curr_regime == INSERT_INCLUDES && filesToInclude.find(file_name) != filesToInclude.end()) { unparseToBuf = removeIncludeStatsAndUnparse(file, file_name, fout_name.c_str(), allIncludeFiles, out_free_form == 1, moduleUsesByFile, moduleDecls, getObjectForFileFromMap(file_name, exctactedModuleStats), toString, false, true); auto itI = filesToInclude.find(file_name); for (auto& [_, incls] : itI->second) for (auto& incl : incls) if (allIncludeFiles.find(incl) != allIncludeFiles.end()) allIncludeFiles.erase(incl); } else { unparseToBuf = removeIncludeStatsAndUnparse(file, file_name, fout_name.c_str(), allIncludeFiles, out_free_form == 1, moduleUsesByFile, moduleDecls, getObjectForFileFromMap(file_name, exctactedModuleStats), toString, (curr_regime == RENAME_INLCUDES), false); // copy includes that have not changed if (folderName != NULL) { int removeSpfDirs = 0; if (curr_regime == REMOVE_DVM_DIRS) removeSpfDirs = 1; else if (curr_regime == REMOVE_DVM_DIRS_TO_COMMENTS) removeSpfDirs = 2; else if (curr_regime == REMOVE_SPF_DIRS) removeSpfDirs = 3; set allIncludeFilesFiltr; set modFiles; for (auto& elem : moduleDecls) modFiles.insert(elem.second); for (auto& elem : allIncludeFiles) { if (modFiles.find(elem) == modFiles.end()) allIncludeFilesFiltr.insert(elem); } allIncludeFiles = allIncludeFilesFiltr; copyIncludes(allIncludeFiles, commentsToInclude, newCopyDeclToIncl, folderName, keepSpfDirs, out_free_form == 1, (curr_regime == RENAME_INLCUDES), removeSpfDirs); } } //restore of restore restoreCorrectedModuleProcNames(file); } return unparseToBuf; } string unparseProjectToString(SgFile *file, const int curr_regime) { set allIncludeFiles; return unparseProjectIfNeed(file, curr_regime, true, "", NULL, allIncludeFiles, true); } static bool runAnalysis(SgProject &project, const int curr_regime, const bool need_to_unparse, const char *newVer = NULL, const char *folderName = NULL) { if (PASSES_DONE_INIT == false) { for (int i = 0; i < EMPTY_PASS; ++i) PASSES_DONE[i] = 0; PASSES_DONE_INIT = true; } if (isDone(curr_regime)) return false; __spf_print(DEBUG_LVL1, "RUN PASS with name %s\n", passNames[curr_regime]); auto toSendStrMessage = string(passNames[curr_regime]); #ifdef _WIN32 sendMessage_1lvl(wstring(L"выполняется проход '") + wstring(toSendStrMessage.begin(), toSendStrMessage.end()) + L"'"); #else sendMessage_1lvl(wstring(L"running pass '") + wstring(toSendStrMessage.begin(), toSendStrMessage.end()) + L"'"); #endif const int n = project.numberOfFiles(); bool verifyOK = true; int internalExit = 0; int countOfTransform = 0; //for process include files set allIncludeFiles; //for insert and extract dirs map templateDeclInIncludes; //for function checker map> functionNames; // ********************************** /// /// FIRST STEP - RUN ANALYSIS BY FILES /// // ********************************** /// auto timeForPass = high_resolution_clock::now(); sgStats.clear(); sgExprs.clear(); SgStatement::cleanStatsByLine(); SgStatement::cleanParentStatsForExprs(); SgStatement::setCurrProcessFile(""); SgStatement::setCurrProcessLine(-1); set allFileNames; for (int i = n - 1; i >= 0; --i) { SgFile *file = &(project.file(i)); allFileNames.insert(file->filename()); updateStatsExprs(current_file_id, file->filename()); } auto rw_analyzer = ReadWriteAnalyzer(allFuncInfo); // doesn't analyze anything until first query int global_err = 0; map same_decls; for (int i = n - 1; i >= 0; --i) { createNeededException(); SgFile *file = &(project.file(i)); toSendStrMessage = file->filename(); #ifdef _WIN32 sendMessage_2lvl(wstring(L"обработка файла '") + wstring(toSendStrMessage.begin(), toSendStrMessage.end()) + L"'"); #else sendMessage_2lvl(wstring(L"processing file '") + wstring(toSendStrMessage.begin(), toSendStrMessage.end()) + L"'"); #endif sendMessage_progress(std::to_wstring((int)(((double)(n - i) / n) * 100))); const char *file_name = file->filename(); __spf_print(DEBUG_LVL1, " Analyzing: %s\n", file_name); if (curr_regime == CONVERT_TO_ENDDO) ConverToEndDo(file, getObjectForFileFromMap(file_name, SPF_messages)); else if (curr_regime == UNROLL_LOOPS) { int err = unrollLoops(file, getObjectForFileFromMap(file_name, loopGraph), getObjectForFileFromMap(file_name, SPF_messages)); if (err != 0) internalExit = -1; } else if (curr_regime == LOOP_ANALYZER_DATA_DIST_S0 || curr_regime == LOOP_ANALYZER_ALIGNS) { vector tmp; loopAnalyzer(file, parallelRegions, createdArrays, tmp, DATA_DISTR, allFuncInfo, declaredArrays, declaratedArraysSt, arrayLinksByFuncCalls, createDefUseMapByPlace(), true, &(loopGraph.find(file_name)->second)); } else if (curr_regime == LOOP_ANALYZER_DATA_DIST_S1 || curr_regime == ONLY_ARRAY_GRAPH) { try { loopAnalyzer(file, parallelRegions, createdArrays, getObjectForFileFromMap(file_name, SPF_messages), DATA_DISTR, allFuncInfo, declaredArrays, declaratedArraysSt, arrayLinksByFuncCalls, createDefUseMapByPlace(), false, &(loopGraph.find(file_name)->second)); } catch (...) { internalExit = 1; } } else if (curr_regime == LOOP_ANALYZER_COMP_DIST) { auto itFound = loopGraph.find(file_name); loopAnalyzer(file, parallelRegions, createdArrays, getObjectForFileFromMap(file_name, SPF_messages), COMP_DISTR, allFuncInfo, declaredArrays, declaratedArraysSt, arrayLinksByFuncCalls, createDefUseMapByPlace(), false, &(itFound->second)); UniteNestedDirectives(itFound->second); } else if (curr_regime == CALL_GRAPH) { auto it = allFuncInfo.find(file_name); if (it == allFuncInfo.end()) { functionAnalyzer(file, allFuncInfo, getObjectForFileFromMap(file_name, loopGraph), getObjectForFileFromMap(file_name, SPF_messages), fullIR); it = allFuncInfo.find(file_name); //init module symbols if (it != allFuncInfo.end()) { for (auto& func : it->second) { SgStatement* currF = func->funcPointer; const auto& tmp = getModuleSymbols(currF); } } } } else if (curr_regime == CALL_GRAPH2) { checkForRecursion(file, allFuncInfo, getObjectForFileFromMap(file_name, SPF_messages)); intentInsert(getObjectForFileFromMap(file_name, allFuncInfo)); insertIntrinsicStat(getObjectForFileFromMap(file_name, allFuncInfo)); } else if (curr_regime == LOOP_GRAPH) loopGraphAnalyzer(file, getObjectForFileFromMap(file_name, loopGraph), getObjectForFileFromMap(file_name, intervals), getObjectForFileFromMap(file_name, SPF_messages), sharedMemoryParallelization); else if (curr_regime == VERIFY_ENDDO) { bool res = EndDoLoopChecker(file, getObjectForFileFromMap(file_name, SPF_messages)); verifyOK &= res; } else if (curr_regime == VERIFY_INCLUDES) { bool res = IncludeChecker(file, file_name, getObjectForFileFromMap(file_name, SPF_messages)); verifyOK &= res; } else if (curr_regime == VERIFY_DVM_DIRS) { bool res = DvmDirectiveChecker(file, dvmDirErrors, keepDvmDirectives, ignoreDvmChecker); verifyOK &= res; if (dvmDirErrors.size() && ignoreDvmChecker == 0) printDvmActiveDirsErrors(); } else if (curr_regime == VERIFY_EQUIVALENCE) { bool res = EquivalenceChecker(file, file_name, parallelRegions, SPF_messages); verifyOK &= res; } else if (curr_regime == VERIFY_OPERATORS) { bool res = OperatorChecker(file, SPF_messages); verifyOK &= res; } else if (curr_regime == VERIFY_FORMAT) { bool res = checkAndMoveFormatOperators(file, getObjectForFileFromMap(file_name, SPF_messages)); verifyOK &= res; } else if (curr_regime == CORRECT_FORMAT_PLACE) checkAndMoveFormatOperators(file, getObjectForFileFromMap(file_name, SPF_messages), false); else if (curr_regime == CREATE_PARALLEL_DIRS || curr_regime == INSERT_PARALLEL_DIRS_NODIST) { auto &loopsInFile = getObjectForFileFromMap(file_name, loopGraph); map depInfoForLoopGraphV; for (auto& elem : depInfoForLoopGraph) depInfoForLoopGraphV[elem.first] = elem.second; createParallelDirs(new File(file), createdDirectives, getObjectForFileFromMap(file_name, SPF_messages), loopsInFile, allFuncInfo, parallelRegions, depInfoForLoopGraphV, arrayLinksByFuncCalls); } else if (curr_regime == INSERT_PARALLEL_DIRS || curr_regime == EXTRACT_PARALLEL_DIRS) { const bool extract = (curr_regime == EXTRACT_PARALLEL_DIRS); insertParallelDirs(file, extract, createdDirectives[file_name], getObjectForFileFromMap(file_name, SPF_messages), templateDeclInIncludes, commentsToInclude, getObjectForFileFromMap(file_name, allFuncInfo), parallelRegions, loopGraph, allFileNames, arrayLinksByFuncCalls, declaredArrays, tableOfUniqNamesByArray); } else if (curr_regime == LOOP_ANALYZER_NODIST) { auto& loopsInFile = getObjectForFileFromMap(file_name, loopGraph); parallizeFreeLoops = 1; sharedMemoryParallelization = 1; loopAnalyzer(file, parallelRegions, createdArrays, getObjectForFileFromMap(file_name, SPF_messages), SHARED_MEMORY_PAR, allFuncInfo, declaredArrays, declaratedArraysSt, arrayLinksByFuncCalls, createDefUseMapByPlace(), false, &(loopsInFile)); UniteNestedDirectives(loopsInFile); } else if (curr_regime == INSERT_SHADOW_DIRS || curr_regime == EXTRACT_SHADOW_DIRS) { const bool extract = (curr_regime == EXTRACT_SHADOW_DIRS); set distrArraysDone; for (int z = 0; z < parallelRegions.size(); ++z) { ParallelRegion *currReg = parallelRegions[z]; DataDirective &dataDirectives = currReg->GetDataDirToModify(); DIST::GraphCSR &reducedG = parallelRegions[z]->GetReducedGraphToModify(); set distrArraysF = fillDistributedArrays(dataDirectives, tableOfUniqNamesByArray, arrayLinksByFuncCalls); set distrArrays; for (auto& elem : distrArraysF) if (distrArraysDone.find(elem) == distrArraysDone.end()) distrArrays.insert(elem); insertShadowSpecToFile(file, file_name, distrArrays, reducedG, commentsToInclude, extract, getObjectForFileFromMap(file_name, SPF_messages), declaredArrays); distrArraysDone.insert(distrArrays.begin(), distrArrays.end()); } if (!extract) { for (int z = 0; z < parallelRegions.size(); ++z) { ParallelRegion* currReg = parallelRegions[z]; DataDirective& dataDirectives = currReg->GetDataDirToModify(); const set distrCommonArrays = fillDistributedArraysD(dataDirectives, tableOfUniqNamesByArray, arrayLinksByFuncCalls, true); insertRealignsBeforeFragments(currReg, file, distrCommonArrays, arrayLinksByFuncCalls); } } else { set dvmDirsToDel; for (SgStatement* st = file->firstStatement(); st; st = st->lexNext()) if (st->variant() == DVM_NEW_VALUE_DIR) dvmDirsToDel.insert(st); for (auto& elem : dvmDirsToDel) elem->deleteStmt(); } } else if (curr_regime == REVERT_SPF_DIRS) { if (keepSpfDirs) revertion_spf_dirs(file, declaredArrays, declaratedArraysSt); else __spf_print(1, " ignore SPF REVERT\n"); } else if (curr_regime == CLEAR_SPF_DIRS) { if (keepSpfDirs) { vector toDel; for (SgStatement *st = file->firstStatement(); st; st = st->lexNext()) { if (isSPF_stat(st)) // except sapfor parallel regions and if attributes dont move { if (st->variant() != SPF_PARALLEL_REG_DIR && st->variant() != SPF_END_PARALLEL_REG_DIR) { if (getAttributes(st->lexNext(), set{ SPF_ANALYSIS_DIR, SPF_PARALLEL_DIR, SPF_TRANSFORM_DIR }).size() > 0) toDel.push_back(st); } } } for (auto &elem : toDel) elem->deleteStmt(); } else __spf_print(1, " ignore CLEAR_SPF_DIRS\n"); } else if (curr_regime == PREPROC_SPF) { bool noError = preprocess_spf_dirs(file, commonBlocks, getObjectForFileFromMap(file_name, SPF_messages), allFileNames, usersDirectives); if (!noError) internalExit = 1; } else if (curr_regime == CHECK_PAR_REG_DIR) { bool noError = check_par_reg_dirs(file, getObjectForFileFromMap(file_name, SPF_messages)); if (!noError) internalExit = 1; } else if (curr_regime == PREPROC_ALLOCATES) preprocess_allocates(file); else if (curr_regime == CORRECT_VAR_DECL) VarDeclCorrecter(file); else if (curr_regime == CREATE_REMOTES) loopAnalyzer(file, parallelRegions, createdArrays, getObjectForFileFromMap(file_name, SPF_messages), REMOTE_ACC, allFuncInfo, declaredArrays, declaratedArraysSt, arrayLinksByFuncCalls, createDefUseMapByPlace(), false, &(loopGraph.find(file_name)->second)); else if (curr_regime == PRIVATE_CALL_GRAPH_STAGE1) FileStructure(file); else if (curr_regime == PRIVATE_CALL_GRAPH_STAGE2) doCallGraph(file); else if (curr_regime == PRIVATE_CALL_GRAPH_STAGE3) { auto itFound = loopGraph.find(file_name); if (itFound != loopGraph.end()) insertSpfAnalysisBeforeParalleLoops(itFound->second); } else if (curr_regime == ARRAY_ACCESS_ANALYSIS_FOR_CORNER) arrayAccessAnalyzer(file, getObjectForFileFromMap(file_name, SPF_messages), declaredArrays, ARRAY_ACC_CORNER); else if (curr_regime == FILL_PAR_REGIONS_LINES) fillRegionLines(file, parallelRegions, getObjectForFileFromMap(file_name, SPF_messages), &(loopGraph[file_name]), &(allFuncInfo[file_name])); else if (curr_regime == FILL_COMMON_BLOCKS) { vector> lines; // lines for getCommonBlocksRef() for (int i = 0; i < file->numberOfFunctions(); ++i) lines.push_back(make_pair(file->functions(i), file->functions(i)->lastNodeOfStmt())); // try to fill from BLOCK DATA SgStatement *st = file->firstStatement(); while (st) { if (st->variant() == BLOCK_DATA) //BLOCK_DATA header { lines.push_back(make_pair(st, st->lastNodeOfStmt())); st = st->lastNodeOfStmt(); } st = st->lexNext(); } for (auto &startEnd : lines) { map> commonBlocksRef; getCommonBlocksRef(commonBlocksRef, startEnd.first, startEnd.second); // adding found common variables for (auto &commonBlockRef : commonBlocksRef) { auto it = commonBlocks.find(commonBlockRef.first); if (it == commonBlocks.end()) it = commonBlocks.insert(it, make_pair(commonBlockRef.first, new CommonBlock(commonBlockRef.first))); int position = 0; for (auto &commonExp : commonBlockRef.second) { vector> newVariables; for (SgExpression *currCommon = commonExp->lhs(); currCommon; currCommon = currCommon->rhs()) newVariables.push_back(make_pair(currCommon->lhs()->symbol(), position++)); it->second->addVariables(file, startEnd.first, newVariables); } } } } else if (curr_regime == VERIFY_COMMON) { bool res = CommonBlockChecker(file, file_name, commonBlocks, SPF_messages); verifyOK &= res; } else if (curr_regime == LOOP_DATA_DEPENDENCIES) doDependenceAnalysisOnTheFullFile(file, 1, 1, 1); else if (curr_regime == REMOVE_DVM_DIRS || curr_regime == REMOVE_DVM_DIRS_TO_COMMENTS || curr_regime == REMOVE_SPF_DIRS) { bool removeDvm = (curr_regime == REMOVE_DVM_DIRS || curr_regime == REMOVE_DVM_DIRS); bool removeSpf = (curr_regime == REMOVE_SPF_DIRS); bool toComment = (curr_regime == REMOVE_DVM_DIRS_TO_COMMENTS); removeDvmSpfDirectives(file, removeDvm, removeSpf, toComment); } else if (curr_regime == REMOVE_DVM_INTERVALS) { vector toDel; for (SgStatement *st = file->firstStatement(); st; st = st->lexNext()) { if (st->variant() == DVM_INTERVAL_DIR) toDel.push_back(st); else if (st->variant() == DVM_ENDINTERVAL_DIR) toDel.push_back(st); else if (st->variant() == DVM_EXIT_INTERVAL_DIR) toDel.push_back(st); } //TODO: move comments for (auto &elem : toDel) elem->deleteStmt(); } else if (curr_regime == SUBST_EXPR) { expressionAnalyzer(file, defUseByFunctions, commonBlocks, temporaryAllFuncInfo, subs_parallelRegions); // analyze again for substituted expressions arrayAccessAnalyzer(file, getObjectForFileFromMap(file_name, SPF_messages), declaredArrays, ARRAY_ACC_CORNER); } else if (curr_regime == REVERT_SUBST_EXPR) revertReplacements(file->filename(), true); else if (curr_regime == CREATE_NESTED_LOOPS) { map mapFuncInfo; createMapOfFunc(allFuncInfo, mapFuncInfo); map depInfoForLoopGraphV; for (auto& elem : depInfoForLoopGraph) depInfoForLoopGraphV[elem.first] = elem.second; auto itFound = loopGraph.find(file_name); if (itFound != loopGraph.end()) for (int i = 0; i < itFound->second.size(); ++i) createNestedLoops(itFound->second[i], depInfoForLoopGraphV, mapFuncInfo, getObjectForFileFromMap(file_name, SPF_messages)); } else if (curr_regime == GET_ALL_ARRAY_DECL) { int err = getAllDeclaredArrays(file, declaredArrays, declaratedArraysSt, SPF_messages, subs_parallelRegions, distrStateFromGUI); if (err != 0) internalExit = -1; } else if (curr_regime == FILE_LINE_INFO) { SgStatement *st = file->firstStatement(); while (st) { string fName = st->fileName(); int line = st->lineNumber(); auto it = lineInfo.find(fName); if (it == lineInfo.end()) it = lineInfo.insert(it, make_pair(fName, 0)); it->second = std::max(it->second, line); if (isSPF_stat(st)) { auto itD = dirsInfo.find(fName); if (itD == dirsInfo.end()) itD = dirsInfo.insert(itD, make_pair(fName, make_pair(set(), set()))); itD->second.first.insert(line); } if (isDVM_stat(st)) { auto itD = dirsInfo.find(fName); if (itD == dirsInfo.end()) itD = dirsInfo.insert(itD, make_pair(fName, make_pair(set(), set()))); itD->second.second.insert(line); } st = st->lexNext(); } } else if (curr_regime == BUILD_INCLUDE_DEPENDENCIES) { auto fileIt = includeDependencies.find(file_name); if (fileIt == includeDependencies.end()) fileIt = includeDependencies.insert(fileIt, make_pair(file_name, map>())); set modFiles; for (auto& elem : moduleDecls) modFiles.insert(elem.second); SgStatement* lastFromFile = NULL; for (SgStatement *st = file->firstStatement(); st; st = st->lexNext()) { if (st->variant() == MODULE_STMT && modFiles.find(st->fileName()) != modFiles.end()) st = st->lastNodeOfStmt(); else { if (strcmp(file_name, st->fileName())) { if (lastFromFile == NULL) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); fileIt->second[lastFromFile->lineNumber()].insert(st->fileName()); } else lastFromFile = st; } } if (inlcudeAllFiles) if (fileIt->second.size()) filesToInclude[file_name] = fileIt->second; } else if (curr_regime == REMOVE_AND_CALC_SHADOW) { map mapFuncInfo; createMapOfFunc(allFuncInfo, mapFuncInfo); devourShadowByRemote(file, mapFuncInfo, getObjectForFileFromMap(file_name, loopGraph), arrayLinksByFuncCalls); } else if (curr_regime == TRANSFORM_SHADOW_IF_FULL) transformShadowIfFull(file, arrayLinksByFuncCalls); else if (curr_regime == MACRO_EXPANSION) doMacroExpand(file, getObjectForFileFromMap(file_name, SPF_messages)); else if (curr_regime == REVERSE_CREATED_NESTED_LOOPS) { auto itFound = loopGraph.find(file_name); if (itFound != loopGraph.end()) reverseCreatedNestedLoops(file->filename(), itFound->second); } else if (curr_regime == CONVERT_ASSIGN_TO_LOOP) // And MOVE COMPLITE DECLARATION as ASSIGN OPERATORS convertFromAssignToLoop(file, subs_parallelRegions, getObjectForFileFromMap(file_name, SPF_messages), arrayLinksByFuncCalls); else if (curr_regime == CONVERT_LOOP_TO_ASSIGN) { if (PASSES_DONE[CONVERT_ASSIGN_TO_LOOP]) restoreAssignsFromLoop(file, arrayLinksByFuncCalls); else __spf_print(1, "skip CONVERT_LOOP_TO_ASSIGN\n"); } else if (curr_regime == CALCULATE_STATS_SCHEME) processFileToPredict(file, getObjectForFileFromMap(file_name, allPredictorStats)); else if (curr_regime == DEF_USE_STAGE1) constructDefUseStep1(file, defUseByFunctions, temporaryAllFuncInfo, getObjectForFileFromMap(file_name, SPF_messages)); else if (curr_regime == DEF_USE_STAGE2) constructDefUseStep2(file, defUseByFunctions); else if (curr_regime == RESTORE_LOOP_FROM_ASSIGN) restoreConvertedLoopForParallelLoops(file, arrayLinksByFuncCalls); else if (curr_regime == RESTORE_LOOP_FROM_ASSIGN_BACK) restoreConvertedLoopForParallelLoops(file, arrayLinksByFuncCalls, true); else if (curr_regime == FILL_PARALLEL_REG_IR) fillRegionLines(file, subs_parallelRegions, getObjectForFileFromMap(file_name, SPF_messages)); else if (curr_regime == CALL_GRAPH_IR) { auto it = allFuncInfo_IR.find(file_name); if (it == allFuncInfo_IR.end()) { vector tmp; map> tmp1; functionAnalyzer(file, allFuncInfo_IR, tmp, getObjectForFileFromMap(file_name, SPF_messages), tmp1); } } else if (curr_regime == ADD_TEMPL_TO_USE_ONLY) fixUseOnlyStmt(file, parallelRegions); else if (curr_regime == GCOV_PARSER) parse_gcovfile(file, consoleMode == 1 ? file_name : "./visualiser_data/gcov/" + string(file_name), getObjectForFileFromMap(file_name, gCovInfo), keepFiles); else if(curr_regime == PRIVATE_ARRAYS_EXPANSION) { auto founded = loopGraph.find(file->filename()); if (founded != loopGraph.end()) privateArraysResizing(file, founded->second, getObjectForFileFromMap(file_name, SPF_messages), countOfTransform, usersDirectives, true); } else if (curr_regime == PRIVATE_ARRAYS_SHRINKING_ANALYSIS) { auto founded = loopGraph.find(file->filename()); if (founded != loopGraph.end()) analyzeShrinking(file, founded->second, getObjectForFileFromMap(file_name, SPF_messages), usersDirectives); } else if (curr_regime == PRIVATE_ARRAYS_SHRINKING) { auto founded = loopGraph.find(file->filename()); if (founded != loopGraph.end()) privateArraysResizing(file, founded->second, getObjectForFileFromMap(file_name, SPF_messages), countOfTransform, usersDirectives, false); } else if(curr_regime == LOOPS_SPLITTER) { auto founded = loopGraph.find(file->filename()); if (founded != loopGraph.end()) { int err = splitLoops(file, founded->second, getObjectForFileFromMap(file_name, SPF_messages), depInfoForLoopGraph, commonBlocks, allFuncInfo, countOfTransform); if (err != 0) internalExit = err; } } else if(curr_regime == LOOPS_COMBINER) { auto founded = loopGraph.find(file->filename()); if (founded != loopGraph.end()) { int err = combineLoops(file, founded->second, getObjectForFileFromMap(file_name, SPF_messages), inOnlyForloopOnPlace, depInfoForLoopGraph, countOfTransform); if (err != 0) internalExit = err; } } else if (curr_regime == PRIVATE_REMOVING_ANALYSIS) { auto itFound = loopGraph.find(file->filename()); if (itFound != loopGraph.end()) removePrivatesAnalysis(file_name, itFound->second, getObjectForFileFromMap(file_name, SPF_messages), usersDirectives, commonBlocks, allFuncInfo); } else if (curr_regime == PRIVATE_REMOVING) removePrivates(file_name, getObjectForFileFromMap(file_name, SPF_messages), commonBlocks, allFuncInfo, countOfTransform); else if (curr_regime == CREATE_INTER_TREE) { vector include_functions; createInterTree(file, getObjectForFileFromMap(file_name, intervals), removeNestedIntervals, getObjectForFileFromMap(file_name, SPF_messages)); assignCallsToFile(consoleMode == 1 ? file_name : "./visualiser_data/gcov/" + string(file_name), getObjectForFileFromMap(file_name, intervals)); removeNodes(intervals_threshold, getObjectForFileFromMap(file_name, intervals), include_functions); } else if (curr_regime == INSERT_INTER_TREE) insertIntervals(file, getObjectForFileFromMap(file_name, intervals)); else if (curr_regime == VERIFY_FUNC_DECL) { bool res = FunctionsChecker(file, functionNames, SPF_messages); verifyOK &= res; } else if (curr_regime == RESTORE_COPIES) restoreCopies(file); else if (curr_regime == SET_TO_ALL_DECL_INIT_ZERO) setAllDeclsWithInitZero(file); else if (curr_regime == DUMP_DECLS_WITH_INIT) dumpAllDeclsWithInit(file, (i == n - 1)); else if (curr_regime == CREATE_CHECKPOINTS) createCheckpoints(file, commonBlocks, filesInfo, getObjectForFileFromMap(file_name, allFuncInfo)); else if (curr_regime == CONVERT_SAVE_TO_MODULE) convertSaveToModule(file); else if (curr_regime == PROCESS_IO) preprocessOpenOperators(file, filesInfo); else if (curr_regime == CONVERT_STRUCTURES_TO_SIMPLE) replaceStructuresToSimpleTypes(file); else if (curr_regime == PURE_INTENT_INSERT) intentInsert(getObjectForFileFromMap(file_name, allFuncInfo)); else if (curr_regime == REMOVE_UNUSED_FUNCTIONS) { auto funcsForFile = getObjectForFileFromMap(file_name, allFuncInfo); for (auto& func : funcsForFile) if (!func->doNotInline && func->deadFunction) func->funcPointer->GetOriginal()->setUnparseIgnore(true); } else if (curr_regime == GROUP_ACTUAL_AND_REMOTE) groupActualAndRemote(file); else if (curr_regime == GROUP_ACTUAL_AND_REMOTE_RESTORE) groupActualAndRemote(file, true); else if (curr_regime == RENAME_SYMBOLS) runRenameSymbolsByFiles(file, &project); else if (curr_regime == REMOVE_OMP_DIRS) { for (SgStatement* st = file->firstStatement(); st; st = st->lexNext()) removeOmpDir(st); } else if (curr_regime == PARSE_OMP_DIRS) parseOmpDirectives(file, getObjectForFileFromMap(file_name, SPF_messages)); else if (curr_regime == REMOVE_COMMENTS) { for (SgStatement* st = file->firstStatement(); st; st = st->lexNext()) if (st->comments()) st->delComments(); } else if (curr_regime == GET_MIN_MAX_BLOCK_DIST) getMaxMinBlockDistribution(file, min_max_block); else if (curr_regime == CONVERT_TO_C) covertToC(file); else if (curr_regime == SET_IMPLICIT_NONE) implicitCheck(file, getObjectForFileFromMap(file_name, dvmDirErrors)); else if (curr_regime == INSERT_NO_DISTR_FLAGS_FROM_GUI) addPrivatesToArraysFromGUI(file, declaredArrays, distrStateFromGUI); else if (curr_regime == REMOVE_DEAD_CODE) { auto funcsForFile = getObjectForFileFromMap(file_name, allFuncInfo); for (auto& func : funcsForFile) if(func->funcPointer->variant() != ENTRY_STAT) countOfTransform += removeDeadCode(func->funcPointer, allFuncInfo, commonBlocks); } else if (curr_regime == TEST_PASS) { //test pass } unparseProjectIfNeed(file, curr_regime, need_to_unparse, newVer, folderName, allIncludeFiles); } // end of FOR by files if (internalExit < 0) throw -1; set applyFor = { LOOPS_SPLITTER, LOOPS_COMBINER, PRIVATE_REMOVING, PRIVATE_ARRAYS_EXPANSION, PRIVATE_ARRAYS_SHRINKING, REMOVE_DEAD_CODE }; if ((countOfTransform == 0 || internalExit > 0) && applyFor.find(curr_regime) != applyFor.end()) { SgStatement* mainUnit = findMainUnit(&project, SPF_messages); checkNull(mainUnit, convertFileName(__FILE__).c_str(), __LINE__); getObjectForFileFromMap(mainUnit->fileName(), SPF_messages).push_back(Messages(ERROR, mainUnit->lineNumber(), R197, L"Transformation cannot be performed - nothing to change", 2023)); __spf_print(1, "%d Transformation cannot be performed - nothing to change, count of transform %d, err %d\n", mainUnit->lineNumber(), countOfTransform, internalExit); throw -11; } sendMessage_2lvl(wstring(L"")); // ********************************** /// /// SECOND AGGREGATION STEP /// // ********************************** /// SgStatement::setCurrProcessFile(""); SgStatement::setCurrProcessLine(-1); if (curr_regime == LOOP_ANALYZER_DATA_DIST_S2 || curr_regime == ONLY_ARRAY_GRAPH) { if (curr_regime == ONLY_ARRAY_GRAPH) keepFiles = 1; if (sharedMemoryParallelization) { for (auto& byFile : loopGraph) for (auto& loop : byFile.second) if (loop->region) loop->reduceAccessGraph(); } else { set usedArraysAcrossRegions; for (int i = 0; i < parallelRegions.size(); ++i) { ParallelRegion* currReg = parallelRegions[i]; DIST::GraphCSR& G = currReg->GetGraphToModify(); DIST::GraphCSR& reducedG = currReg->GetReducedGraphToModify(); DIST::Arrays& allArrays = currReg->GetAllArraysToModify(); usedArraysAcrossRegions.insert(allArrays.GetArrays().begin(), allArrays.GetArrays().end()); if (currReg->HasUserDvmDirs()) { set alignedArrays; set addedArrays; bool error = false; if (currReg->GetUsersDirecites(DVM_REALIGN_DIR)->size()) { bool ret = buildGraphFromUserDirectives(*(currReg->GetUsersDirecites(DVM_REALIGN_DIR)), G, allArrays, arrayLinksByFuncCalls, alignedArrays, addedArrays, allFuncInfo); error = error || ret; alignedArrays.insert(addedArrays.begin(), addedArrays.end()); } if (currReg->GetUsersDirecites(DVM_ALIGN_DIR)->size()) { bool ret = buildGraphFromUserDirectives(*(currReg->GetUsersDirecites(DVM_ALIGN_DIR)), G, allArrays, arrayLinksByFuncCalls, alignedArrays, addedArrays, allFuncInfo); error = error || ret; alignedArrays.insert(addedArrays.begin(), addedArrays.end()); } if (currReg->GetUsersDirecites(DVM_DISTRIBUTE_DIR)->size()) error = false; if (error == false) error = (currReg->GetUsersDirecites(DVM_REALIGN_DIR)->size() == 0) && (currReg->GetUsersDirecites(DVM_ALIGN_DIR)->size() == 0) && (currReg->GetUsersDirecites(DVM_DISTRIBUTE_DIR)->size() == 0); if (error) { wstring messageR, messageE; __spf_printToLongBuf(messageE, L"Can not build align graph from user's DVM directives in this region"); __spf_printToLongBuf(messageR, R67); for (auto& lines : currReg->GetAllLines()) { const auto& vecLines = lines.second; const string& fileName = lines.first; auto& messages = getObjectForFileFromMap(fileName.c_str(), SPF_messages); for (auto& line : vecLines) { if ((line.stats.first && line.stats.second) || currReg->GetId() == 0) { messages.push_back(Messages(ERROR, line.lines.first, messageR, messageE, 1036)); __spf_print(1, "Can not build align graph from user's DVM directives in this region in '%s': %d\n", fileName.c_str(), line.lines.first); } } } printInternalError(convertFileName(__FILE__).c_str(), __LINE__); } else { currReg->ClearUserDirs(); set varsToClear = { DVM_DYNAMIC_DIR, DVM_INHERIT_DIR }; removeStatementsFromAllproject(varsToClear); //TODO: need to add template clones, so remove all dirs! // clear user directives from all loops for (auto& byFile : loopGraph) { if (SgFile::switchToFile(byFile.first) == -1) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); for (auto& loop : byFile.second) loop->clearUserDirectives(); } // clear other dirs set varsToClear1 = { ACC_REGION_DIR, ACC_END_REGION_DIR, ACC_ACTUAL_DIR, ACC_GET_ACTUAL_DIR, DVM_SHADOW_DIR, DVM_REALIGN_DIR, DVM_REDISTRIBUTE_DIR, DVM_VAR_DECL, HPF_TEMPLATE_STAT, DVM_REMOTE_ACCESS_DIR }; removeStatementsFromAllproject(varsToClear1); } } G.SetMaxAvailMemory(currentAvailMemory); G.ChangeQuality(QUALITY, SPEED); reducedG.SetMaxAvailMemory(currentAvailMemory); DIST::createOptimalDistribution(G, reducedG, allArrays, currReg->GetId(), (curr_regime == ONLY_ARRAY_GRAPH)); compliteArrayUsage(allArrays, createdArrays, arrayLinksByFuncCalls, tableOfUniqNamesByArray); } remoteNotUsedArrays(createdArrays, usedArraysAcrossRegions, arrayLinksByFuncCalls); for (map::iterator it = shortFileNames.begin(); it != shortFileNames.end(); it++) __spf_print(1, " %s -> %s\n", it->first.c_str(), it->second.c_str()); set idxToDel; for (int z = 0; z < parallelRegions.size(); ++z) { if (parallelRegions[z]->GetAllArrays().GetArrays().size() == 0) { __spf_print(1, " CAN NOT FIND ARRAYS FOR DISTRIBUTION for parallel region '%s'\n", parallelRegions[z]->GetName().c_str()); if (parallelRegions[z]->GetId() == 0) // DEFAULT { wstring bufE, bufR; __spf_printToLongBuf(bufE, L"Can not find arrays or free loops for distribution in this project"); __spf_printToLongBuf(bufR, R130); for (auto& funcByFile : allFuncInfo) { vector& fileM = getObjectForFileFromMap(funcByFile.first.c_str(), SPF_messages); for (auto& func : funcByFile.second) { auto stat = func->funcPointer->GetOriginal(); if (stat->variant() == PROG_HEDR) fileM.push_back(Messages(ERROR, stat->lineNumber(), bufR, bufE, 3010)); } } } else { wstring bufE, bufR; __spf_printToLongBuf(bufE, L"Can not find arrays or free loops for distribution in this region"); __spf_printToLongBuf(bufR, R131); for (auto& linesByFile : parallelRegions[z]->GetAllLines()) { vector& fileM = getObjectForFileFromMap(linesByFile.first.c_str(), SPF_messages); for (auto& lines : linesByFile.second) if (!lines.isImplicit()) fileM.push_back(Messages(ERROR, lines.lines.first, bufR, bufE, 3010)); } } idxToDel.insert(z); } } vector newParReg; for (int z = 0; z < parallelRegions.size(); ++z) { if (idxToDel.find(z) != idxToDel.end()) { ParallelRegion* regToDel = parallelRegions[z]; #ifdef _WIN32 removeFromCollection(parallelRegions[z]); #endif delete parallelRegions[z]; } else newParReg.push_back(parallelRegions[z]); } parallelRegions.clear(); parallelRegions = newParReg; if (parallelRegions.size() == 0) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); } } else if (curr_regime == CALL_GRAPH) { if (keepFiles) { map V; vector E; CreateCallGraphViz("_callGraph.txt", allFuncInfo, V, E); } findDeadFunctionsAndFillCalls(allFuncInfo, SPF_messages); createLinksBetweenFormalAndActualParams(allFuncInfo, arrayLinksByFuncCalls, declaredArrays, SPF_messages, keepFiles); propagateWritesToArrays(allFuncInfo); updateFuncInfo(allFuncInfo); uniteIntervalsBetweenProcCalls(intervals, allFuncInfo); auto positions = buildLocationOfGraph(allFuncInfo, 10); if (keepFiles) { CreateFuncInfo("_funcInfo.txt", allFuncInfo); saveIntervals("_intervals_united.txt", intervals); } updateLoopIoAndStopsByFuncCalls(loopGraph, allFuncInfo); moveAllocatesInterproc(arrayLinksByFuncCalls); removeDistrStateFromDeadFunctions(allFuncInfo, declaredArrays); propagateArrayFlags(arrayLinksByFuncCalls, declaredArrays, SPF_messages); } else if (curr_regime == CALL_GRAPH2) { map allFuncs; createMapOfFunc(allFuncInfo, allFuncs); completeFillOfArrayUsageBetweenProc(loopGraph, allFuncInfo); fixTypeOfArrayInfoWithCallGraph(declaredArrays, allFuncs); //for each file of graphLoop for (auto &loopGraphInFile : loopGraph) { LoopChecker checker(loopGraphInFile.second); checker.updateLoopGraph(allFuncs); } detectCopies(allFuncInfo); fillInterfaceBlock(allFuncInfo); intentInsertToInterfaces(allFuncInfo); createInterfacesForAssumedSize(allFuncInfo); //this call is only for testing //setPureStatus(allFuncInfo); //add conflict messages for all loops for (auto byFile : loopGraph) { map allLoops; createMapLoopGraph(byFile.second, allLoops); for (auto& loop : allLoops) { if (loop.second->hasLimitsToParallel()) { loop.second->addConflictMessages(&SPF_messages[loop.second->fileName]); __spf_print(1, " added conflict messages to loop on line %d\n", loop.second->lineNum); } } } if (keepFiles) printArrayInfo("_arrayInfo.txt", declaredArrays); } else if (curr_regime == INSERT_SHADOW_DIRS) { for (auto &comment : commentsToInclude) { if (consoleMode && !folderName) { __spf_print(1, " write to <%s_%s> file\n", comment.first.c_str(), newVer); insertDistributionToFile(comment.first.c_str(), (comment.first + "_" + string(newVer)).c_str(), comment.second); } else { if (folderName) { __spf_print(1, " write to <%s> file\n", (string(folderName) + "/" + string(comment.first)).c_str()); insertDistributionToFile(comment.first.c_str(), (string(folderName) + "/" + string(comment.first)).c_str(), comment.second); } /*else { __spf_print(1, " write to <%s> file\n", (string(comment.first)).c_str()); insertDistributionToFile(it->first.c_str(), (string(comment.first)).c_str(), comment.second); }*/ } } // copy includes that have not changed if (folderName != NULL) copyIncludes(allIncludeFiles, commentsToInclude, newCopyDeclToIncl, folderName, out_free_form == 1, keepSpfDirs, false); } else if (curr_regime == EXTRACT_SHADOW_DIRS) commentsToInclude.clear(); else if (curr_regime == VERIFY_ENDDO || curr_regime == VERIFY_INCLUDES || curr_regime == VERIFY_DVM_DIRS || curr_regime == VERIFY_EQUIVALENCE || curr_regime == VERIFY_COMMON || curr_regime == VERIFY_FUNC_DECL || curr_regime == VERIFY_OPERATORS || curr_regime == VERIFY_FORMAT) { if (verifyOK == false) throw(-1); } else if (curr_regime == PRIVATE_ANALYSIS_SPF) { SgStatement *mainUnit = findMainUnit(&project, SPF_messages); if (mainUnit) { //PrivateAnalyzerForMain(mainUnit); Private_Vars_Analyzer(mainUnit); } else { for (int i = n - 1; i >= 0; --i) { createNeededException(); SgFile *file = &(project.file(i)); auto funcForFile = allFuncInfo.find(file->filename()); if (funcForFile != allFuncInfo.end()) PrivateAnalyzer(file, funcForFile->second); } } } else if (curr_regime == CREATE_TEMPLATE_LINKS) { vector result; set arraysDone; if (sharedMemoryParallelization) { bool wasDone = false; for (int z = 0; z < parallelRegions.size(); ++z) { DIST::Arrays& allArrays = parallelRegions[z]->GetAllArraysToModify(); if (ALGORITHMS_DONE[CREATE_DISTIBUTION][z] == 0) recalculateArraySizes(arraysDone, allArrays.GetArrays(), arrayLinksByFuncCalls, allFuncInfo); if (ALGORITHMS_DONE[CREATE_DISTIBUTION][z] != 0) wasDone = true; ALGORITHMS_DONE[CREATE_DISTIBUTION][z] = 1; ALGORITHMS_DONE[CREATE_ALIGNS][z] = 1; } if (parallelRegions.size() == 0) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); if (!wasDone) { for (auto& byFile : loopGraph) for (auto& loop : byFile.second) loop->createVirtualTemplateLinks(arrayLinksByFuncCalls, SPF_messages, sharedMemoryParallelization > 0); //add dummy array DataDirective& dataDirectives = parallelRegions[0]->GetDataDirToModify(); DIST::Arrays& allArrays = parallelRegions[0]->GetAllArraysToModify(); for (auto& byFile : loopGraph) { map allLoops; createMapLoopGraph(byFile.second, allLoops); bool found = false; for (auto& loop : allLoops) { auto& tmp = loop.second->getDataDir(); if (tmp.distrRules.size() != 0) { dataDirectives.distrRules.push_back(tmp.distrRules[0]); DIST::Array* firstTempl = dataDirectives.distrRules.back().first; firstTempl->DeprecateAllDims(); allArrays.AddArrayToGraph(firstTempl); firstTempl->ChangeName("dvmh_additional_parallel"); found = true; break; } } if (found) break; } } } else { for (int z = 0; z < parallelRegions.size(); ++z) { DIST::GraphCSR& reducedG = parallelRegions[z]->GetReducedGraphToModify(); DIST::Arrays& allArrays = parallelRegions[z]->GetAllArraysToModify(); DataDirective& dataDirectives = parallelRegions[z]->GetDataDirToModify(); if (ALGORITHMS_DONE[CREATE_DISTIBUTION][z] == 0) { //recalculate array sizes after expression substitution recalculateArraySizes(arraysDone, allArrays.GetArrays(), arrayLinksByFuncCalls, allFuncInfo); createDistributionDirs(reducedG, allArrays, dataDirectives, SPF_messages, arrayLinksByFuncCalls, sharedMemoryParallelization > 0); ALGORITHMS_DONE[CREATE_DISTIBUTION][z] = 1; } if (ALGORITHMS_DONE[CREATE_ALIGNS][z] == 0) { if (keepFiles) { char fName[256]; sprintf(fName, "_graph_reduced_with_templ_reg%lld.txt", parallelRegions[z]->GetId()); reducedG.CreateGraphWiz(fName, vector>(), allArrays, true); } createAlignDirs(reducedG, allArrays, dataDirectives, parallelRegions[z]->GetId(), arrayLinksByFuncCalls, SPF_messages); ALGORITHMS_DONE[CREATE_ALIGNS][z] = 1; } set toCheck; for (auto& arrayP : dataDirectives.GenAlignsRules(NULL)) toCheck.insert(arrayP.alignArray); internalExit = checkCommonInMainUnit(project, SPF_messages, toCheck, true); __spf_print(1, "*** FOR PARALLEL REGION '%s':\n", parallelRegions[z]->GetName().c_str()); result = dataDirectives.GenAlignsRules(); for (int i = 0; i < result.size(); ++i) __spf_print(1, " %s\n", result[i].c_str()); /*shiftAlignRulesForTemplates(allArrays.GetArrays(), parallelRegions[z]->GetId(), dataDirectives, arrayLinksByFuncCalls); __spf_print(1, "*** MODIFY FOR PARALLEL REGION '%s':\n", parallelRegions[z]->GetName().c_str()); result = dataDirectives.GenAlignsRules(); for (int i = 0; i < result.size(); ++i) __spf_print(1, " %s\n", result[i].c_str()); printf("");*/ } } } else if (curr_regime == FILE_LINE_INFO) { int allLineSum = 0; for (auto &elem : lineInfo) allLineSum += elem.second; __spf_print(1, " All lines in project %d\n", allLineSum); } else if (curr_regime == FILL_PAR_REGIONS_LINES) { fillRegionLinesStep2(parallelRegions, allFuncInfo, &loopGraph); if (ignoreDvmChecker == 1 && parallelRegions.size() == 1 && parallelRegions[0]->GetName() == "DEFAULT" && dvmDirErrors.size()) { printDvmActiveDirsErrors(); throw(-1); } checkCountOfIter(loopGraph, allFuncInfo, SPF_messages); calculateLinesOfCode(parallelRegions); if (keepFiles) { printLoopGraph("_loopGraph_with_reg.txt", loopGraph, true); printParalleRegions("_parallelRegions.txt", parallelRegions); } } if (curr_regime == SET_IMPLICIT_NONE) { size_t total_s = 0; for (auto& err : dvmDirErrors) total_s += err.second.size(); if (total_s && ignoreDvmChecker == 0) { printDvmActiveDirsErrors(); throw(-1); } } else if (curr_regime == FILL_PAR_REGIONS) { fillRegionIntervals(parallelRegions); fillRegionFunctions(parallelRegions, allFuncInfo); fillRegionArrays(parallelRegions, allFuncInfo, commonBlocks); bool noError = checkRegions(parallelRegions, allFuncInfo, SPF_messages); if (!noError) internalExit = 1; if (keepFiles) { int err = printCheckRegions("_checkRegions.txt", parallelRegions, allFuncInfo); if (err == -1) internalExit = 1; err = printCheckRegions(NULL, parallelRegions, allFuncInfo); if (err == -1) internalExit = 1; } } else if (curr_regime == CHECK_ARGS_DECL) { bool error = checkArgumentsDeclaration(&project, allFuncInfo, parallelRegions, SPF_messages); if (error) internalExit = 1; } else if (curr_regime == RESOLVE_PAR_REGIONS) { bool error = resolveParRegions(parallelRegions, allFuncInfo, SPF_messages, sharedMemoryParallelization, newCopyDeclToIncl); if (error) internalExit = 1; } else if (curr_regime == EXPAND_EXTRACT_PAR_REGION) { bool error = expandExtractReg(std::get<0>(inData), std::get<1>(inData), std::get<2>(inData), parallelRegions, getObjectForFileFromMap(std::get<0>(inData).c_str(), SPF_messages), !std::get<3>(inData)); if (error) internalExit = 1; } else if (curr_regime == REMOVE_DIST_ARRAYS_FROM_IO) replaceDistributedArraysInIO(parallelRegions, allFuncInfo, SPF_messages, newCopyDeclToIncl); else if (curr_regime == LOOP_GRAPH) { if (keepFiles) printLoopGraph("_loopGraph.txt", loopGraph); //printf("%s\n", convertToJson(loopGraph).dump(2).c_str()); } else if (curr_regime == FILL_COMMON_BLOCKS) { if (keepFiles) printCommonBlocks("_commonBlocks.txt", commonBlocks); } else if (curr_regime == REVERT_SUBST_EXPR) PASSES_DONE[SUBST_EXPR] = 0; else if (curr_regime == REVERT_SUBST_EXPR_RD) { revertSubstitutions(); PASSES_DONE[SUBST_EXPR_RD] = 0; } else if (curr_regime == INSERT_PARALLEL_DIRS || curr_regime == EXTRACT_PARALLEL_DIRS) { bool cond = (folderName != NULL) || (consoleMode) || (!consoleMode && curr_regime == EXTRACT_PARALLEL_DIRS); if (cond && sharedMemoryParallelization == 0) { //insert template declaration to main program const bool extract = (curr_regime == EXTRACT_PARALLEL_DIRS); for (int i = n - 1; i >= 0; --i) { createNeededException(); SgFile *file = &(project.file(i)); if (file->mainProgram()) { string fileName = file->filename(); auto itDep = includeDependencies.find(fileName); //TODO: split by functions set includedToThisFile; if (itDep != includeDependencies.end()) { for (auto& [_, incls] : itDep->second) { for (auto& incl : incls) { auto comm = commentsToInclude.find(incl); if (comm != commentsToInclude.end()) for (auto& allComm : comm->second) includedToThisFile.insert(allComm.second.begin(), allComm.second.end()); } } } for (int z = 0; z < parallelRegions.size(); ++z) { ParallelRegion *currReg = parallelRegions[z]; const DataDirective &dataDirectives = currReg->GetDataDir(); const vector ¤tVariant = currReg->GetCurrentVariant(); const DIST::Arrays &allArrays = currReg->GetAllArrays(); const vector distrRules = dataDirectives.GenRule(currentVariant); const vector> distrRulesSt = dataDirectives.GenRule(currentVariant, 0); insertTempalteDeclarationToMainFile(file, dataDirectives, templateDeclInIncludes, distrRules, distrRulesSt, allArrays, extract, currReg->GetId(), includedToThisFile); } break; } } } if (curr_regime == EXTRACT_PARALLEL_DIRS) { for (int i = n - 1; i >= 0; --i) { SgFile* file = &(project.file(i)); SgStatement* st = file->firstStatement(); vector declares; while (st) { if (st->variant() == ACC_DECLARE_DIR) declares.push_back(st); st = st->lexNext(); } for (auto& elem : declares) elem->deleteStmt(); } } } else if (curr_regime == DEF_USE_STAGE1) { if (keepFiles) printDefUseSets("_defUseList.txt", defUseByFunctions); } else if (curr_regime == LOOP_ANALYZER_DATA_DIST_S0) excludeArraysFromDistribution(arrayLinksByFuncCalls, declaredArrays, loopGraph, parallelRegions, SPF_messages, createdArrays, sharedMemoryParallelization); else if (curr_regime == LOOP_ANALYZER_DATA_DIST_S1) { for (int z = 0; z < parallelRegions.size(); ++z) { ParallelRegion *currReg = parallelRegions[z]; auto graph = currReg->GetGraph(); __spf_print(1, "STAT: par reg %s: requests %d, miss %d, V = %d, E = %d\n", currReg->GetName().c_str(), graph.getCountOfReq(), graph.getCountOfMiss(), graph.GetNumberOfV(), graph.GetNumberOfE()); } } else if (curr_regime == PRINT_PAR_REGIONS_ERRORS) { bool error = checkRegionsResolving(parallelRegions, allFuncInfo, commonBlocks, SPF_messages, sharedMemoryParallelization); if (error) internalExit = 1; } else if (curr_regime == FILL_PARALLEL_REG_IR) { fillRegionLinesStep2(subs_parallelRegions, allFuncInfo_IR); clearRegionStaticData(); } else if (curr_regime == CALL_GRAPH_IR) findDeadFunctionsAndFillCalls(allFuncInfo_IR, SPF_messages, true); else if (curr_regime == GET_ALL_ARRAY_DECL) { bool hasNonDefaultReg = false; for (auto &elem : subs_parallelRegions) if (elem->GetName() != "DEFAULT") hasNonDefaultReg = true; if (hasNonDefaultReg) { for (auto array : declaredArrays) { if (array.second.first->GetRegionsName().size() == 0) array.second.first->SetDistributeFlag(DIST::NO_DISTR); else if (array.second.first->GetRegionsName().size() == 1) { string regName = *array.second.first->GetRegionsName().begin(); convertToLower(regName); if (regName == "default") array.second.first->SetDistributeFlag(DIST::NO_DISTR); } } } if (ignoreArrayDistributeState) for (auto array : declaredArrays) array.second.first->SetDistributeFlag(DIST::NO_DISTR); } else if (curr_regime == GCOV_PARSER) { parseTimesDvmStatisticFile((consoleMode == 1) ? string("statistic.txt") : "./visualiser_data/statistic/" + string("statistic.txt"), intervals); //fixed count, devide by value from PROG_HEDR SgStatement* mainUnit = findMainUnit(&project, SPF_messages); if (mainUnit->variant() != PROG_HEDR) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); const int line = mainUnit->lineNumber(); auto itF = gCovInfo.find(mainUnit->fileName()); if (itF != gCovInfo.end()) { auto itL = itF->second.find(line); if (itL != itF->second.end()) { auto totalExec = itL->second.getExecutedCount(); if (totalExec > 1) { for (auto& byFile : gCovInfo) for (auto& byLine : byFile.second) byLine.second.setExecutedCount(byLine.second.getExecutedCount() / totalExec); } } } } else if (curr_regime == PREDICT_SCHEME) { int maxSizeDist = 0; for (int z = 0; z < parallelRegions.size(); ++z) { const DataDirective &dataDirectives = parallelRegions[z]->GetDataDir(); const vector ¤tVariant = parallelRegions[z]->GetCurrentVariant(); auto &tmp = dataDirectives.distrRules; vector> currentVar; for (int z1 = 0; z1 < currentVariant.size(); ++z1) currentVar.push_back(make_pair(tmp[z1].first, &tmp[z1].second[currentVariant[z1]])); for (auto &elem : currentVar) { DIST::Array *array = elem.first; const DistrVariant *var = elem.second; int countBlock = 0; for (int z = 0; z < var->distRule.size(); ++z) if (var->distRule[z] == dist::BLOCK) ++countBlock; maxSizeDist = std::max(maxSizeDist, countBlock); } } SpfInterval *mainIterval = getMainInterval(&project, intervals, SPF_messages); topologies.clear(); if (maxSizeDist) { const int procNum = 8; //TODO: //topologies = getTopologies(procNum, maxSizeDist); throw -10; const int countOfTop = topologies.size(); if (countOfTop < 0) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); for (auto &inter : intervals) initTimeForIntervalTree(countOfTop, inter.second); for (int z = 0; z < parallelRegions.size(); ++z) { const DataDirective &dataDirectives = parallelRegions[z]->GetDataDir(); const vector ¤tVariant = parallelRegions[z]->GetCurrentVariant(); DIST::Arrays &allArrays = parallelRegions[z]->GetAllArraysToModify(); auto &tmp = dataDirectives.distrRules; vector> currentVar; for (int z1 = 0; z1 < currentVariant.size(); ++z1) currentVar.push_back(make_pair(tmp[z1].first, &tmp[z1].second[currentVariant[z1]])); map parallelDirs; vector, pair>> allSingleRemotes; for (int i = n - 1; i >= 0; --i) { SgFile *file = &(project.file(i)); auto fountInfo = findAllDirectives(file, getObjectForFileFromMap(file->filename(), loopGraph), parallelRegions[z]->GetId()); parallelDirs.insert(fountInfo.begin(), fountInfo.end()); auto fountRem = findAllSingleRemotes(file, parallelRegions[z]->GetId(), parallelRegions); allSingleRemotes.insert(allSingleRemotes.end(), fountRem.begin(), fountRem.end()); } //TODO! //int err = predictScheme(parallelRegions[z], currentVar, allArrays.GetArrays(), parallelDirs, intervals, SPF_messages, allSingleRemotes, maxSizeDist, procNum); /*if (err != 0) internalExit = err;*/ } vector tmp = { mainIterval }; aggregatePredictedTimes(tmp); int idx = 0; int best = -1; double bestSpeedUp = 0; for (auto &top : topologies) { string outStr = ""; for (auto &elem : top) outStr += std::to_string(elem) + " "; double currS = mainIterval->exec_time / mainIterval->predictedTimes[idx]; __spf_print(1, "%d: speed up %f for top. %s\n", idx, currS, outStr.c_str()); if (best == -1 || bestSpeedUp < currS) { bestSpeedUp = currS; best = idx; } ++idx; } __spf_print(1, "best topology %d with speed up %f\n", best, bestSpeedUp); } else for (auto &inter : intervals) initTimeForIntervalTree(0, inter.second); } else if (curr_regime == CREATE_INTER_TREE) { if (keepFiles) saveIntervals("_intervals.txt", intervals); } else if (curr_regime == CREATE_PARALLEL_REGIONS) { SpfInterval* mainInterval = getMainInterval(&project, intervals, SPF_messages); createParallelRegions(&project, mainInterval, gCovInfo, allFuncInfo); } else if (curr_regime == SUBST_EXPR) PASSES_DONE[REVERT_SUBST_EXPR] = 0; else if (curr_regime == SUBST_EXPR_RD) { PASSES_DONE[REVERT_SUBST_EXPR_RD] = 0; performRDSubst(fullIR, commonBlocks, &project); } else if (curr_regime == DUPLICATE_FUNCTIONS) { if (!duplicateFunctions(allFuncInfo, arrayLinksByFuncCalls, SPF_messages)) internalExit = 1; } else if (curr_regime == REMOVE_COPIES) removeCopies(allFuncInfo); else if (curr_regime == ADD_TEMPL_TO_USE_ONLY) correctTemplateModuleDeclaration((folderName == NULL) ? "" : folderName); else if (curr_regime == PURE_COMMON_TO_PARAMS) commonTransfer(allFuncInfo, commonBlocks); else if (curr_regime == PURE_SAVE_TO_PARAMS) saveTransfer(allFuncInfo); else if (curr_regime == PURE_MODULE_TO_PARAMS) moduleTransfer(allFuncInfo); else if (curr_regime == PURE_INTENT_INSERT) setPureStatus(allFuncInfo); else if (curr_regime == SHADOW_GROUPING) { if (staticShadowAnalysis) { //TODO for all parallel regions if (parallelRegions.size() == 1 && parallelRegions[0]->GetName() == "DEFAULT") { for (int z = 0; z < parallelRegions.size(); ++z) GroupShadow(allFuncInfo, loopGraph, parallelRegions[z]->GetAllArraysToModify(), arrayLinksByFuncCalls, commonBlocks); } } else __spf_print(1, "skip\n"); } else if (curr_regime == SWAP_LOOPS) swapLoopsForParallel(loopGraph, SPF_messages, 1); else if (curr_regime == RESTORE_SWAP_LOOPS) swapLoopsForParallel(loopGraph, SPF_messages, -1); else if (curr_regime == CREATE_PARALLEL_DIRS || curr_regime == INSERT_PARALLEL_DIRS_NODIST) filterParallelDirectives(loopGraph, createdDirectives); else if (curr_regime == INLINE_PROCEDURES) callInliner(allFuncInfo, inDataProc, inDataChains, inDataChainsStart, SPF_messages, commonBlocks); else if (curr_regime == INSERT_REGIONS) internalExit = insertDvmhRegions(project, n, parallelRegions, allFuncInfo, loopGraph, rw_analyzer, SPF_messages, arrayLinksByFuncCalls); else if (curr_regime == RENAME_SYMBOLS) runRenameSymbols(&project, commonBlocks); else if (curr_regime == FIND_PARAMETERS) parametersOfProject = findParameters(defUseByFunctions, commonBlocks, allFuncInfo); else if (curr_regime == BUILD_IR) { auto CFG_forFile = buildCFG(commonBlocks, allFuncInfo_IR, SAPFOR::CFG_Settings(0)); for (auto& byFunc : CFG_forFile) fullIR[byFunc.first].insert(fullIR[byFunc.first].end(), byFunc.second.begin(), byFunc.second.end()); // dump if (dumpIR) dumpCFG(fullIR, dumpRD ? true : false); } else if (curr_regime == LIVE_ANALYSIS_IR) { runLiveVariableAnalysis(fullIR); if (dumpLive) doDumpLive(fullIR); } else if (curr_regime == PRIVATE_ANALYSIS_IR) runPrivateVariableAnalysis(loopGraph, fullIR, commonBlocks, SPF_messages); else if (curr_regime == FIX_COMMON_BLOCKS) fixCommonBlocks(allFuncInfo, commonBlocks, &project); else if (curr_regime == GET_MIN_MAX_BLOCK_DIST) __spf_print(1, "GET_MIN_MAX_BLOCK_DIST: %d %d\n", min_max_block.first, min_max_block.second); else if (curr_regime == GET_STATS_FOR_PREDICTOR) { calculateStatsForPredictor(allFuncInfo, gCovInfo); parseDvmDirForPredictor(declaredArrays, commonBlocks, allFuncInfo, gCovInfo); } const float elapsed = duration_cast(high_resolution_clock::now() - timeForPass).count() / 1000.; const float elapsedGlobal = duration_cast(high_resolution_clock::now() - globalTime).count() / 1000.; __spf_print(1, "PROFILE: time for this pass = %f sec (total %f sec)\n", elapsed, elapsedGlobal); sendMessage_2lvl(wstring(L"")); if (internalExit != 0) throw -1; return true; } static int countMaxValuesForParallelVariants(int &maxDims, int &maxDimsIdx, int &maxDimsIdxReg, vector> ¤tVariants) { int lastI; if (consoleMode) { for (int z = 0; z < parallelRegions.size(); ++z) { DataDirective &dataDirectives = parallelRegions[z]->GetDataDirToModify(); for (int i = 0; i < dataDirectives.distrRules.size(); ++i) { const int currDim = (int)dataDirectives.distrRules[0].second.size(); if (maxDims < currDim) { maxDims = currDim; maxDimsIdx = i; maxDimsIdxReg = z; } } } for (int z = 0; z < parallelRegions.size(); ++z) { DataDirective &dataDirectives = parallelRegions[z]->GetDataDirToModify(); currentVariants[z].resize(dataDirectives.distrRules.size()); for (int i = 0; i < dataDirectives.distrRules.size(); ++i) currentVariants[z][i] = 0; parallelRegions[z]->SetCurrentVariant(currentVariants[z]); } lastI = (int)(parallelRegions[maxDimsIdxReg]->GetDataDir().distrRules[maxDimsIdx].second.size()); } else lastI = 1; return lastI; } static string selectAddNameOfVariant(const int i, int maxDimsIdx, int maxDimsIdxReg, vector> ¤tVariants) { char buf[256]; if (consoleMode) { sprintf(buf, "v%d", i + 1); currentVariants[maxDimsIdxReg][maxDimsIdx] = i; if (i != 0) { for (int p = 0; p < parallelRegions.size(); ++p) { DataDirective& dataDirectives = parallelRegions[p]->GetDataDirToModify(); for (int z = 0; z < dataDirectives.distrRules.size(); ++z) { if (z == maxDimsIdx && maxDimsIdxReg == p) continue; currentVariants[p][z] = (currentVariants[p][z] + 1) % dataDirectives.distrRules[z].second.size(); } } } for (int z = 0; z < parallelRegions.size(); ++z) parallelRegions[z]->SetCurrentVariant(currentVariants[z]); } else sprintf(buf, ""); //sprintf(buf, "par"); return buf; } static int lastFailedStatus = 0; static map> lastErrors; static void findFunctionsToInclude(bool needToAddErrors) { if (isDone(FIND_FUNC_TO_INCLUDE)) { if (needToAddErrors && lastFailedStatus != 0) { for (auto& byFile : lastErrors) for (auto& message : byFile.second) SPF_messages[byFile.first].push_back(message); if (lastFailedStatus > 0) throw -5; else throw -6; } return; } lastErrors.clear(); __spf_print(1, "RUN PASS with name FindFunctionsToInclude\n"); map files; for (int i = 0; i < project->numberOfFiles(); ++i) { SgFile *file = &(project->file(i)); files.insert(make_pair(file->filename(), i)); } const char* keepToFile = NULL; if (keepFiles) keepToFile = "_callGraph_withInc.txt"; map> localMessages; int failed = CheckFunctionsToInline(project, files, NULL, allFuncInfo, loopGraph, localMessages, true, arrayLinksByFuncCalls, parallelRegions); for (auto& byFile : localMessages) { for (auto& message : byFile.second) { if (needToAddErrors) SPF_messages[byFile.first].push_back(message); else { if (message.type != ERROR) SPF_messages[byFile.first].push_back(message); else lastErrors[byFile.first].push_back(message); } } } if (failed > 0 && needToAddErrors) throw -5; /*else if (failed < 0) throw -6;*/ lastFailedStatus = failed; } static bool dvmInited = false; void runPass(const int curr_regime, const char *proj_name, const char *folderName, bool root_call) { // create full dependency graph switch (curr_regime) { case LOOPS_SPLITTER: case LOOPS_COMBINER: fullDepGraph = true; break; } // init math functions of FORTRAN if(root_call) { initIntrinsicFunctionNames(); initTags(); InitPassesDependencies(passesDependencies, passesIgnoreStateDone); removalsFromPassesDependencies(passesDependencies, curr_regime); setPassValues(); } if (dvmInited == false) { int t1 = out_free_form; int t2 = out_upper_case; int t3 = out_line_unlimit; initialize(); // init variables in dvm.cpp dvmInited = true; out_free_form = t1; out_upper_case = t2; out_line_unlimit = t3; } if (project == NULL && curr_regime != PARSE_FILES) { project = createProject(proj_name, parallelRegions, subs_parallelRegions, hiddenData, filesNameWithoutExt, moduleUsesByFile, moduleDecls, exctactedModuleStats, printSymbTable); //first check correctness runPass(VERIFY_FUNC_DECL, proj_name, folderName); } //Run dep passes analysis before main pass auto itDep = passesDependencies.find((passes)curr_regime); if (itDep != passesDependencies.end()) for (int k = 0; k < itDep->second.size(); ++k) runPass(itDep->second[k], proj_name, folderName); switch (curr_regime) { case FIND_FUNC_TO_INCLUDE: findFunctionsToInclude(false); break; case FILL_PAR_REGIONS_LINES: if (runAnalysis(*project, curr_regime, false)) { // init flags for all parallel regions if (ALGORITHMS_DONE[0] == NULL) { for (int i = 0; i < EMPTY_ALGO; ++i) { ALGORITHMS_DONE[i] = new int[parallelRegions.size()]; for (int k = 0; k < parallelRegions.size(); ++k) ALGORITHMS_DONE[i][k] = 0; } } } break; case INSERT_PARALLEL_DIRS_NODIST: { sharedMemoryParallelization = 1; string additionalName = (consoleMode && folderName == NULL) ? "__shared" : ""; runAnalysis(*project, CREATE_PARALLEL_DIRS, false); runPass(REVERT_SUBST_EXPR_RD, proj_name, folderName); runPass(CONVERT_LOOP_TO_ASSIGN, proj_name, folderName); runPass(REVERT_SUBST_EXPR_RD, proj_name, folderName); runAnalysis(*project, INSERT_PARALLEL_DIRS, false, consoleMode ? additionalName.c_str() : NULL, folderName); runPass(RESTORE_LOOP_FROM_ASSIGN, proj_name, folderName); runAnalysis(*project, INSERT_REGIONS, false); runAnalysis(*project, REMOVE_COPIES, false); runAnalysis(*project, SWAP_LOOPS, false); if (folderName || consoleMode) runAnalysis(*project, UNPARSE_FILE, true, additionalName.c_str(), folderName); } break; case INSERT_PARALLEL_DIRS: { vector> currentVariants(parallelRegions.size()); int maxDims = 0; int maxDimsIdx = -1; int maxDimsIdxReg = -1; int lastI = 1; if (sharedMemoryParallelization == 0) lastI = countMaxValuesForParallelVariants(maxDims, maxDimsIdx, maxDimsIdxReg, currentVariants); if (genAllVars == 0) lastI = 1; for (int i = 0; i < lastI; ++i) { //if specific variant number is requested, skip all others if (genSpecificVar >= 0 && i != genSpecificVar && sharedMemoryParallelization == 0) continue; string tmpFolder = ""; if (consoleMode && !folderName) { tmpFolder = "p" + to_string(i + 1); bool res = createDirectory(tmpFolder); if (res == false) { __spf_print(1, "can not create folder '%s'\n", tmpFolder.c_str()); tmpFolder = ""; } else folderName = tmpFolder.c_str(); } string additionalName = selectAddNameOfVariant(i, maxDimsIdx, maxDimsIdxReg, currentVariants); runPass(CREATE_PARALLEL_DIRS, proj_name, folderName); runPass(REVERT_SUBST_EXPR_RD, proj_name, folderName); runPass(CONVERT_LOOP_TO_ASSIGN, proj_name, folderName); runAnalysis(*project, INSERT_PARALLEL_DIRS, false, consoleMode ? additionalName.c_str() : NULL, folderName); if (sharedMemoryParallelization == 0) { runPass(CREATE_REMOTES, proj_name, folderName); runPass(REMOVE_AND_CALC_SHADOW, proj_name, folderName); runPass(SHADOW_GROUPING, proj_name, folderName); runPass(TRANSFORM_SHADOW_IF_FULL, proj_name, folderName); runAnalysis(*project, INSERT_SHADOW_DIRS, false, consoleMode ? additionalName.c_str() : NULL, folderName); } runPass(RESTORE_LOOP_FROM_ASSIGN, proj_name, folderName); if (sharedMemoryParallelization == 0) runPass(ADD_TEMPL_TO_USE_ONLY, proj_name, folderName); runAnalysis(*project, INSERT_REGIONS, false); if (sharedMemoryParallelization == 0) runPass(GROUP_ACTUAL_AND_REMOTE, proj_name, folderName); runAnalysis(*project, CALCULATE_STATS_SCHEME, false); //TODO: need to rewrite this to new algo /*if (!folderName && !consoleMode || predictOn) runAnalysis(*project, PREDICT_SCHEME, false); */ runAnalysis(*project, REMOVE_COPIES, false); runAnalysis(*project, SWAP_LOOPS, false); if (folderName || consoleMode) runAnalysis(*project, UNPARSE_FILE, true, (tmpFolder == "") ? additionalName.c_str() : "", folderName); runAnalysis(*project, RESTORE_SWAP_LOOPS, false); // restore runAnalysis(*project, RESTORE_COPIES, false); runPass(EXTRACT_PARALLEL_DIRS, proj_name, folderName); runPass(EXTRACT_SHADOW_DIRS, proj_name, folderName); runPass(REVERSE_CREATED_NESTED_LOOPS, proj_name, folderName); runPass(CLEAR_SPF_DIRS, proj_name, folderName); runPass(RESTORE_LOOP_FROM_ASSIGN_BACK, proj_name, folderName); if (sharedMemoryParallelization == 0) runPass(GROUP_ACTUAL_AND_REMOTE_RESTORE, proj_name, folderName); //clear shadow grouping clearAllocatedShadowNodes(); for (auto& funcbyFile : allFuncInfo) { for (auto& func : funcbyFile.second) { if (func->shadowTreeStart) delete func->shadowTreeStart; if (func->shadowTreeEnd) delete func->shadowTreeEnd; func->shadowTreeStart = func->shadowTreeEnd = NULL; func->allShadowNodes.clear(); } } //clear template clones && region status for (auto& loopByFile : loopGraph) { for (auto& loop : loopByFile.second) { loop->clearForSwap(); if (loop->directive) loop->directive->cloneOfTemplate = ""; loop->inDvmhRegion = 0; loop->propagateDvmhRegion(0); } } FILE* templatesInfo = NULL; if (consoleMode && withTemplateInfo) { string file = (folderName) ? (folderName + string("/")) : ""; file += "templates_info_"; file += to_string(i + 1); file += ".txt"; templatesInfo = fopen(file.c_str(), "w"); if (templatesInfo == NULL) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); } for (int z = 0; z < parallelRegions.size(); ++z) { if (consoleMode && withTemplateInfo) fprintf(templatesInfo, "FOR region %s\n", parallelRegions[z]->GetName().c_str()); const DataDirective& dataDirectives = parallelRegions[z]->GetDataDir(); const vector& currentVariant = parallelRegions[z]->GetCurrentVariant(); auto& tmp = dataDirectives.distrRules; for (int z1 = 0; z1 < currentVariant.size(); ++z1) { if (tmp[z1].first->IsTemplate()) { if (consoleMode && withTemplateInfo) { SgExpression* rule = new SgExpression(EXPR_LIST); tmp[z1].second[currentVariant[z1]].GenRule(new File(current_file), new Expression(rule), tmp[z1].first->GetNewTemplateDimsOrder()); fprintf(templatesInfo, "%s(%s)\n", tmp[z1].first->GetShortName().c_str(), rule->unparse()); } tmp[z1].first->ClearTemplateClones(); } } } if (tmpFolder != "" && consoleMode) folderName = NULL; if (consoleMode && withTemplateInfo) fclose(templatesInfo); } } break; case CREATE_NESTED_LOOPS: case CONVERT_TO_ENDDO: case CORRECT_CODE_STYLE: case INSERT_INCLUDES: case REMOVE_DVM_DIRS: case REMOVE_DVM_DIRS_TO_COMMENTS: case REMOVE_SPF_DIRS: case PRIVATE_ARRAYS_EXPANSION: case PRIVATE_ARRAYS_SHRINKING: case UNROLL_LOOPS: case INSERT_INTER_TREE: case REMOVE_DVM_INTERVALS: case SET_TO_ALL_DECL_INIT_ZERO: case CREATE_CHECKPOINTS: case PURE_INTENT_INSERT: case REMOVE_OMP_DIRS_TRANSFORM: case REMOVE_COMMENTS: case INSERT_NO_DISTR_FLAGS_FROM_GUI: case PRIVATE_REMOVING: case RENAME_INLCUDES: runAnalysis(*project, curr_regime, true, "", folderName); break; case INLINE_PROCEDURES: runAnalysis(*project, curr_regime, false); if (folderName) runAnalysis(*project, UNPARSE_FILE, true, "", folderName); else __spf_print(1, "can not run UNPARSE_FILE - folder name is null\n"); break; case UNPARSE_FILE: if (folderName) runAnalysis(*project, curr_regime, true, "", folderName); else __spf_print(1, "can not run UNPARSE_FILE - folder name is null\n"); break; case CHECK_FUNC_TO_INCLUDE: findFunctionsToInclude(true); break; // all these cases run UNPARSE_FILE after case REMOVE_DIST_ARRAYS_FROM_IO: case RENAME_SYMBOLS: case RESOLVE_PAR_REGIONS: case CREATE_PARALLEL_REGIONS: case DUPLICATE_FUNCTIONS: case CONVERT_STRUCTURES_TO_SIMPLE: case PURE_COMMON_TO_PARAMS: case PURE_SAVE_TO_PARAMS: case PURE_MODULE_TO_PARAMS: case REMOVE_UNUSED_FUNCTIONS: case INSERT_REGIONS: case LOOPS_SPLITTER: case LOOPS_COMBINER: case FIX_COMMON_BLOCKS: case TEST_PASS: case SET_IMPLICIT_NONE: runAnalysis(*project, curr_regime, false); case SUBST_EXPR_RD_AND_UNPARSE: case SUBST_EXPR_AND_UNPARSE: case REMOVE_DEAD_CODE_AND_UNPARSE: if (folderName) runAnalysis(*project, UNPARSE_FILE, true, "", folderName); else __spf_print(1, "can not run UNPARSE_FILE - folder name is null\n"); break; case ADD_TEMPL_TO_USE_ONLY: runAnalysis(*project, curr_regime, false, "", folderName); break; case PRIVATE_ANALYSIS_SPF: if (staticPrivateAnalysis) runAnalysis(*project, curr_regime, false); break; case PARSE_FILES: { int err = parseFiles(proj_name, filesCompilationOrder, parseForInlining); if (err != 0) throw err; } break; default: runAnalysis(*project, curr_regime, false); break; } } int main(int argc, char **argv) { int leakMemDump = 0; bool withDel = false; try { setPassValues(); consoleMode = 1; QUALITY = 100; SPEED = 100; bool printed = false; for (int i = 0; i < argc; ++i) { const char* curr_arg = argv[i]; if (string(curr_arg) == "-client") { printVersion("[SAPFOR]: "); printed = true; break; } } if (!printed) printVersion(); const char *proj_name = "dvm.proj"; const char *folderName = NULL; int curr_regime = EMPTY_PASS; int numVar = 0; out_free_form = 0; // F90 style out out_upper_case = 0; out_line_unlimit = 0; bool printText = false; int serverPort = -1; bool printPasses = false; for (int i = 0; i < argc; ++i) { const char *curr_arg = argv[i]; switch (curr_arg[0]) { case '-': if (string(curr_arg) == "-threshold") { i++; intervals_threshold = atoi(argv[i]); } else if (string(curr_arg) == "-removeNestedIntervals") removeNestedIntervals = true; else if (string(curr_arg) == "-p") { i++; proj_name = argv[i]; } else if (string(curr_arg) == "-pass") { i++; curr_regime = atoi(argv[i]); } else if (string(curr_arg) == "-passN") { i++; curr_regime = getPassCode(argv[i]); printf("code for pass %s is %d\n", argv[i], curr_regime); } else if (string(curr_arg) == "-passInfo") printPasses = true; else if (string(curr_arg) == "-q1") { i++; QUALITY = atoi(argv[i]); if (QUALITY <= 0 || QUALITY > 100) { __spf_print(1, "QUALITY must be in [0..100] interval, set default value 100"); QUALITY = 100; } } else if (string(curr_arg) == "-q2") { i++; SPEED = atoi(argv[i]); if (SPEED <= 0 || SPEED > 100) { __spf_print(1, "SPEED must be in [0..100] interval, set default value 100"); SPEED = 100; } } else if (string(curr_arg) == "-tinfo") withTemplateInfo = true; else if (string(curr_arg) == "-t") // deprecated { i++; int par = atoi(argv[i]); if (par == 13) curr_regime = INSERT_PARALLEL_DIRS; } else if (curr_arg[1] == 'h') ;// printHelp(passNames, EMPTY_PASS); else if (string(curr_arg) == "-leak") { leakMemDump = 1; #if _WIN32 && _DEBUG _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF); #endif } else if (string(curr_arg) == "-f90") { out_free_form = 1; out_line_unlimit = 1; } else if (string(curr_arg) == "-sh") staticShadowAnalysis = 1; else if (string(curr_arg) == "-shWidth") { i++; maxShadowWidth = atoi(argv[i]); } else if (string(curr_arg) == "-priv") staticPrivateAnalysis = 1; else if (string(curr_arg) == "-keep") keepFiles = 1; else if (string(curr_arg) == "-pred") predictOn = 1; else if (string(curr_arg) == "-keepSPF") keepSpfDirs = 1; else if (string(curr_arg) == "-keepDVM") keepDvmDirectives = 1; else if (string(curr_arg) == "-allVars") genAllVars = 1; else if (string(curr_arg) == "-delLeak") withDel = true; else if (string(curr_arg) == "-Var" || string(curr_arg) == "-var") { //User sees variants starting from 1, internally they start from 0 i++; genAllVars = 1; genSpecificVar = atoi(argv[i]) - 1; } else if (string(curr_arg) == "-F") { i++; folderName = argv[i]; } else if (string(curr_arg) == "-print") printText = true; else if (string(curr_arg) == "-useDvm") ignoreDvmChecker = 1; else if (string(curr_arg) == "-passTree") InitPassesDependencies(passesDependencies, passesIgnoreStateDone, true); else if (string(curr_arg) == "-ver" || string(curr_arg) == "-Ver") exit(0); else if (string(curr_arg) == "-freeLoops") parallizeFreeLoops = 1; else if (string(curr_arg) == "-autoArray") parallizeFreeLoops = 1; else if (string(curr_arg) == "-parse") parseFiles(argc - (i + 1), argv + (i + 1)); else if (string(curr_arg) == "-pppa") pppaAnalyzer(argc - i, argv + i); else if (string(curr_arg) == "-fdvm") convertFiles(argc - i, argv + i); else if (string(curr_arg) == "-mpi" || string(curr_arg) == "-shared") { sharedMemoryParallelization = 1; ignoreArrayDistributeState = true; } else if (string(curr_arg) == "-client") { runAsClient = true; withDel = false; consoleMode = false; i++; if (i < argc) serverPort = atoi(argv[i]); break; } else if (string(curr_arg) == "-inlineH") { inDataProc.clear(); inDataChainsStart.clear(); inDataChains.clear(); i++; inDataChainsStart.insert(argv[i]); } else if (string(curr_arg) == "-inlineI") { inDataProc.clear(); inDataChainsStart.clear(); inDataChains.clear(); i++; string funcName = argv[i++]; int line = atoi(argv[i++]); string fileName = argv[i]; inDataProc.push_back(make_tuple(funcName, fileName, line)); } else if (string(curr_arg) == "-dumpIR") dumpIR = 1; else if (string(curr_arg) == "-dumpRD") dumpRD = 1; else if (string(curr_arg) == "-dumpLive") dumpLive = 1; else if (string(curr_arg) == "-ignoreDistArray") ignoreArrayDistributeState = true; else if (string(curr_arg) == "-debSh") debSh = 1; else if (string(curr_arg) == "-noLogo") noLogo = true; else if (string(curr_arg) == "-includeAll") inlcudeAllFiles = true; else if (string(curr_arg) == "-printSymbTable") printSymbTable = true; break; default: break; } } if (sharedMemoryParallelization == 1) { keepDvmDirectives = 0; ignoreIO = 1; if (curr_regime == INSERT_PARALLEL_DIRS) { printf("[SAPFOR]: deprecated regime: pass == INSERT_PARALLEL_DIRS and -mpi option. Instead, you need to use INSERT_PARALLEL_DIRS_NODIST pass\n"); exit(-1); } } if (curr_regime == INSERT_PARALLEL_DIRS_NODIST) ignoreArrayDistributeState = true; if (runAsClient) { printf("[SAPFOR]: Started as client with server port %d\n", serverPort); RunSapforAsClient(serverPort); } else { if (curr_regime == EMPTY_PASS) printHelp(passNames, printPasses ? EMPTY_PASS : -1); runPass(curr_regime, proj_name, folderName, true); if (printText) runPass(UNPARSE_FILE, proj_name, folderName, true); } } catch (...) { printStackTrace(); printf("exception occurred\n"); for (auto& byFile : SPF_messages) { for (auto& message : byFile.second) { string toPrint = ""; for (int z = 0; z < message.engMessage.size(); ++z) toPrint += message.engMessage[z]; string type; if (message.type == WARR) type = "WARR"; else if (message.type == ERROR) type = "ERROR"; else if (message.type == NOTE) type = "NOTE"; else type = "UNKN"; printf("%s - [#%d: %s: line %d]: %s\n", type.c_str(), message.group, byFile.first.c_str(), message.line, toPrint.c_str()); } } } deleteAllAllocatedData(withDel); #if _WIN32 && _DEBUG if (leakMemDump) { printf("START PRINT\n"); _CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE); _CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDOUT); _CrtDumpMemoryLeaks(); } #endif return 0; }