merged mastser

src/DirectiveProcessing/directive_creator_base.cpp

@@ -1601,7 +1601,7 @@ static bool matchParallelAndDist(const pair<DIST::Array*, const DistrVariant*>&
         parallelOn = NULL;
         for (auto& refOn : realRefs)
         {
-            linkWithTempl = parallelOn->GetLinksWithTemplate(regionId);
+            linkWithTempl = refOn->GetLinksWithTemplate(regionId);
             int z = 0;
             for (int k = 0; k < linkWithTempl.size(); ++k)
                 if (linkWithTempl[k] != -1)

src/Distribution/Array.h

@@ -106,7 +106,7 @@ namespace Distribution
         MAP<STRING, SET<int>> usagePlaces;
 
         VECTOR<bool> mappedDims;
-        VECTOR<bool> depracateToDistribute;
+        VECTOR<bool> deprecateToDistribute;
 
         bool ompThreadPrivate;
         bool privateInLoop;
@@ -171,7 +171,7 @@ namespace Distribution
             sizes.resize(dimSize);
             sizesExpr.resize(dimSize);
             mappedDims.resize(dimSize);
-            depracateToDistribute.resize(dimSize);
+            deprecateToDistribute.resize(dimSize);
 
             for (int z = 0; z < dimSize; ++z)
             {
@@ -179,7 +179,7 @@ namespace Distribution
                 PAIR<int, int> initVal = std::make_pair(0, 0);
                 sizesExpr[z] = std::make_pair(std::make_pair((Expression*)NULL, initVal), std::make_pair((Expression*)NULL, initVal));
                 mappedDims[z] = false;
-                depracateToDistribute[z] = false;
+                deprecateToDistribute[z] = false;
             }
                         
             GenUniqKey();
@@ -214,7 +214,7 @@ namespace Distribution
             uniqKey = copy.uniqKey;
             containsInRegions = copy.containsInRegions;
             mappedDims = copy.mappedDims;
-            depracateToDistribute = copy.depracateToDistribute;
+            deprecateToDistribute = copy.deprecateToDistribute;
             ompThreadPrivate = copy.ompThreadPrivate;
             privateInLoop = copy.privateInLoop;
             inEquivalence = copy.inEquivalence;
@@ -226,7 +226,7 @@ namespace Distribution
             int countToRem = 0;
             for (int z = 0; z < dimSize; ++z)
             {
-                if (!mappedDims[z] || depracateToDistribute[z])
+                if (!mappedDims[z] || deprecateToDistribute[z])
                 {
                     needToRemove = true;
                     countToRem++;
@@ -246,19 +246,19 @@ namespace Distribution
 
             for (int z = 0; z < dimSize; ++z)
             {
-                if (mappedDims[z] && !depracateToDistribute[z])
+                if (mappedDims[z] && !deprecateToDistribute[z])
                 {
                     newSizes.push_back(sizes[z]);
                     newSizesExpr.push_back(sizesExpr[z]);
                     newMappedDims.push_back(mappedDims[z]);
-                    newDepr.push_back(depracateToDistribute[z]);
+                    newDepr.push_back(deprecateToDistribute[z]);
                 }
             }
 
             sizes = newSizes;
             sizesExpr = newSizesExpr;
             mappedDims = newMappedDims;
-            depracateToDistribute = newDepr;
+            deprecateToDistribute = newDepr;
             dimSize = (int)sizes.size();
 
             return false;
@@ -426,49 +426,6 @@ namespace Distribution
 
         void ClearShadowSpecs() { allShadowSpecs.clear(); }
 
-        //TODO: to remove
-        STRING toString()
-        {
-            STRING retVal = "";
-            retVal += TO_STR(id);
-            retVal += "#" + name;
-            retVal += "#" + shortName;
-            retVal += "#" + TO_STR(dimSize);
-            retVal += "#" + TO_STR(typeSize);
-            retVal += "#" + TO_STR(isNonDistribute);
-            
-            retVal += "#" + TO_STR(locationPos.first);
-            retVal += "#" + locationPos.second;
-
-            retVal += "#" + TO_STR(sizes.size());
-            for (int i = 0; i < sizes.size(); ++i)
-                retVal += "#" + TO_STR(sizes[i].first) + "#" + TO_STR(sizes[i].second);
-
-            retVal += "#" + TO_STR(depracateToDistribute.size());
-            for (int i = 0; i < depracateToDistribute.size(); ++i)
-                retVal += "#" + TO_STR((int)depracateToDistribute[i]);
-
-            retVal += "#" + TO_STR(mappedDims.size());
-            for (int i = 0; i < mappedDims.size(); ++i)
-                retVal += "#" + TO_STR((int)mappedDims[i]);
-
-            retVal += "#" + TO_STR(templateInfo.size());
-            for (auto it = templateInfo.begin(); it != templateInfo.end(); ++it)
-                retVal += "#" + TO_STR(it->first) + it->second->toString();
-            
-            retVal += "#" + TO_STR((int)isTemplFlag);
-            retVal += "|" + TO_STR((int)isLoopArrayFlag);
-            retVal += "|" + TO_STR(declPlaces.size());
-
-            for (auto &place : declPlaces)
-                retVal += "|" + place.first + "|" + TO_STR(place.second);
-
-            retVal += "|" + TO_STR(containsInRegions.size());
-            for (auto &reg : containsInRegions)
-                retVal += "|" + reg;
-            return retVal;
-        }
-
         JSON toJson()
         {
             JSON retVal;
@@ -476,6 +433,7 @@ namespace Distribution
             retVal["id"] = (int64_t)id;
             retVal["name"] = name;
             retVal["shortName"] = shortName;
+            retVal["packedAddress"] = std::to_string((long long)this);
 
             retVal["dimSize"] = dimSize;
             retVal["typeSize"] = typeSize;
@@ -487,9 +445,9 @@ namespace Distribution
             retVal["isLoopArrayFlag"] = (int)isLoopArrayFlag;
 
             JSON deprToDist = nlohmann::json::array();
-            for (int i = 0; i < depracateToDistribute.size(); ++i)
+            for (int i = 0; i < deprecateToDistribute.size(); ++i)
-                deprToDist.push_back((int)depracateToDistribute[i]);
+                deprToDist.push_back((int)deprecateToDistribute[i]);
-            retVal["depracateToDist"] = deprToDist;
+            retVal["deprecateToDist"] = deprToDist;
 
             JSON mappedDimsJ = nlohmann::json::array();
             for (int i = 0; i < mappedDims.size(); ++i)
@@ -647,13 +605,13 @@ namespace Distribution
         {
             if (dim >= dimSize)
                 return;
-            depracateToDistribute[dim] = value;
+            deprecateToDistribute[dim] = value;
         }
 
         void DeprecateAllDims()
         {
             for (int dim = 0; dim < dimSize; ++dim)
-                depracateToDistribute[dim] = true;
+                deprecateToDistribute[dim] = true;
         }
 
         bool IsDimDepracated(const int dim) const
@@ -663,9 +621,9 @@ namespace Distribution
             else
             {
                 if (templateDimsOrder.size() == 0)
-                    return depracateToDistribute[dim];
+                    return deprecateToDistribute[dim];
                 else
-                    return depracateToDistribute[templateDimsOrder[dim]];
+                    return deprecateToDistribute[templateDimsOrder[dim]];
             }
         }
         
@@ -673,11 +631,11 @@ namespace Distribution
         {
             bool ret = true;
             for (int z = 0; z < dimSize; ++z)
-                ret = ret && depracateToDistribute[z];
+                ret = ret && deprecateToDistribute[z];
             return ret;
         }
 
-        const VECTOR<bool>& GetDeprecetedDims() const { return depracateToDistribute; }
+        const VECTOR<bool>& GetDeprecetedDims() const { return deprecateToDistribute; }
 
         int GetTypeSize() const { return typeSize; }
 

src/Distribution/DvmhDirective.cpp

@@ -199,15 +199,6 @@ static set<string> fillUsedSymbols(SgStatement *loop)
     return usedS;
 }
 
-static string correctSymbolModuleName(const string& origFull)
-{
-    auto it = origFull.find("::");
-    if (it == string::npos)
-        return origFull;
-    else
-        return origFull.substr(it + 2);
-}
-
 static SgStatement* getModuleScope(const string& origFull, vector<SgStatement*>& moduleList, SgStatement *local)
 {
     auto it = origFull.find("::");
@@ -959,7 +950,7 @@ ParallelDirective::genDirective(File* file, const vector<pair<DIST::Array*, cons
                 SgSymbol* redS;
                 string clearName = correctSymbolModuleName(red);
                 if (clearName != red)
-                    redS = getNameInLocation(parentFunc, clearName, getModuleScope(red, moduleList, parentFunc)->symbol()->identifier());
+                    redS = getNameInLocation(parentFunc, red, getModuleScope(red, moduleList, parentFunc)->symbol()->identifier());
                 else
                     redS = findSymbolOrCreate(file, clearName, NULL, parentFunc);
 
@@ -1017,12 +1008,12 @@ ParallelDirective::genDirective(File* file, const vector<pair<DIST::Array*, cons
                 string clearName2 = correctSymbolModuleName(get<1>(list));
 
                 if (clearName1 != get<0>(list))
-                    redS1 = getNameInLocation(parentFunc, clearName1, getModuleScope(get<0>(list), moduleList, parentFunc)->symbol()->identifier());
+                    redS1 = getNameInLocation(parentFunc, get<0>(list), getModuleScope(get<0>(list), moduleList, parentFunc)->symbol()->identifier());
                 else
                     redS1 = findSymbolOrCreate(file, clearName1, NULL, parentFunc);
 
                 if (clearName2 != get<1>(list))
-                    redS2 = getNameInLocation(parentFunc, clearName2, getModuleScope(get<1>(list), moduleList, parentFunc)->symbol()->identifier());
+                    redS2 = getNameInLocation(parentFunc, get<1>(list), getModuleScope(get<1>(list), moduleList, parentFunc)->symbol()->identifier());
                 else
                     redS2 = findSymbolOrCreate(file, clearName2, NULL, parentFunc);
 

src/Distribution/DvmhDirectiveBase.cpp

@@ -12,6 +12,8 @@
 #include "../Distribution/GraphCSR.h"
 #include "../Utils/errors.h"
 #include "../Utils/utils.h"
+#include "../Utils/json.hpp"
+
 #include "../GraphCall/graph_calls_func.h"
 
 using std::vector;
@@ -27,6 +29,8 @@ using std::make_pair;
 using std::min;
 using std::max;
 
+using nlohmann::json;
+
 template<typename setT>
 static void uniteSets(const set<setT> &first, const set<setT> &second, set<setT> &result)
 {
@@ -732,30 +736,34 @@ string AlignRuleBase::GenRuleBase() const
     return retVal;
 }
 
-string AlignRuleBase::toString()
+json AlignRuleBase::toJson()
 {
-    string retVal = "";
+    json retVal;
 
-    if (alignArray)
+    retVal["packedAlignArrayAddress"] = alignArray ? std::to_string((long long)alignArray) : std::to_string((long long)-1);
-        retVal += "#" + std::to_string((long long)alignArray);
+    retVal["packedAlignWithAddress"] = alignWith ? std::to_string((long long)alignWith) : std::to_string((long long)-1);
-    else
-        retVal += "#-1";
 
-    if (alignWith)
+    json alignRules = json::array();
-        retVal += "#" + std::to_string((long long)alignWith);
+    for (auto& rule : alignRule)
-    else
+    {
-        retVal += "#-1";
+        json pair;
+        pair["key"] = rule.first;
+        pair["value"] = rule.second;
+        alignRules.push_back(pair);
+    }
+    retVal["alignRule"] = alignRules;
 
-    retVal += "#" + std::to_string(alignRule.size());
+    json alignRuleWithJ = json::array();
-    for (int i = 0; i < alignRule.size(); ++i)
+    for (auto& [dimNum, AB] : alignRuleWith)
-        retVal += "#" + std::to_string(alignRule[i].first) + "#" + std::to_string(alignRule[i].second);
+    {
-
+        json tuple;
-    retVal += "#" + std::to_string(alignRuleWith.size());
+        tuple["dimNum"] = dimNum;
-    for (int i = 0; i < alignRuleWith.size(); ++i)
+        tuple["a"] = AB.first;
-        retVal += "#" + std::to_string(alignRuleWith[i].first)
+        tuple["b"] = AB.second;
-        + "#" + std::to_string(alignRuleWith[i].second.first)
-        + "#" + std::to_string(alignRuleWith[i].second.second);
 
+        alignRuleWithJ.push_back(tuple);
+    }
+    retVal["alignRuleWith"] = alignRuleWithJ;
     return retVal;
 }
 

src/Distribution/DvmhDirectiveBase.h

@@ -3,6 +3,8 @@
 #include <vector>
 #include <string>
 
+#include "../Utils/json.hpp"
+
 typedef enum lang : int { LANG_C, LANG_F } language;
 typedef enum dist : int { BLOCK, NONE } distType;
 typedef std::pair<std::pair<int, int>, std::pair<int, int>> attrType;
@@ -48,7 +50,7 @@ public:
 
 public:
     std::string GenRuleBase() const;
-    std::string toString();
+    nlohmann::json toJson();
 
 public:
     DIST::Array *alignArray;

src/ParallelizationRegions/ParRegions.h

@@ -11,6 +11,8 @@
 #include "../Distribution/Distribution.h"
 #include "../Utils/AstWrapper.h"
 
+#include "../Utils/json.hpp"
+
 #if __SPF
 #include "../Utils/SgUtils.h"
 #endif
@@ -253,35 +255,42 @@ public:
         return retVal;
     }
 
-    std::string toString()
+    nlohmann::json toJson()
     {
-        std::string retVal = "";
+        nlohmann::json retVal;
 
-        retVal += "#" + std::to_string(regionId);
+        retVal["packedRegionId"] = std::to_string(regionId);
-        retVal += "#" + originalName;
+        retVal["originalName"] = originalName;
-        retVal += "#" + std::to_string(lines.size());
 
-        for (auto it = lines.begin(); it != lines.end(); ++it)
+        nlohmann::json arrays = nlohmann::json::array();
+        for (auto& array : allArrays.GetArrays())
+            arrays.push_back(array->toJson());
+        retVal["packedArrays"] = arrays;
+
+        nlohmann::json linesInfo = nlohmann::json::array();
+        for (auto& [file, linesByFile] : lines)
         {
-            retVal += "|" + it->first + "|";
+            nlohmann::json linesRegs;
-            retVal += std::to_string(it->second.size());
+            nlohmann::json lines = nlohmann::json::array();
-            for (int i = 0; i < it->second.size(); ++i)
+
-                retVal += "#" + std::to_string(it->second[i].lines.first) + "#" + std::to_string(it->second[i].lines.second);
+            for (auto& elem : linesByFile)
+            {
+                JSON pair;
+                pair["key"] = elem.lines.first;
+                pair["value"] = elem.lines.second;
+                lines.push_back(pair);
             }
+            linesRegs["file"] = file;
+            linesRegs["lines"] = lines;
 
-        const std::set<DIST::Array*> &arrays = allArrays.GetArrays();
+            linesInfo.push_back(linesRegs);
-        retVal += "#" + std::to_string(arrays.size());
-
-        //create map<array_address, DIST::Array_toString()> 
-        for (auto it = arrays.begin(); it != arrays.end(); ++it)
-        {
-            retVal += "#" + std::to_string((long long)(*it));
-            retVal += "#" + (*it)->toString();
         }
+        retVal["regionsLines"] = linesInfo;
 
-        retVal += "#" + std::to_string(dataDirectives.alignRules.size());
+        nlohmann::json alignRules = nlohmann::json::array();
-        for (int i = 0; i < dataDirectives.alignRules.size(); ++i)
+        for (auto& rule : dataDirectives.alignRules)
-            retVal += dataDirectives.alignRules[i].toString();
+            alignRules.push_back(rule.toJson());
+        retVal["alignRules"] = alignRules;
 
         return retVal;
     }

src/Predictor/PredictScheme.h

@@ -2,6 +2,7 @@
 #include <vector>
 #include "dvm.h"
 #include "../GraphCall/graph_calls.h"
+#include "../Utils/json.hpp"
 
 class ParallelStats
 {
@@ -11,10 +12,10 @@ public:
         RemoteCount = ShadowCount = ReductionCount = AcrossCount = 0;
     }
      
-    int RemoteCount;
+    int RemoteCount = 0;
-    int ShadowCount;
+    int ShadowCount = 0;
-    int ReductionCount;
+    int ReductionCount = 0;
-    int AcrossCount;
+    int AcrossCount = 0;
 };
 
 class PredictorStats
@@ -27,28 +28,32 @@ public:
     }
      
     ParallelStats ParallelStat;
-    int ParallelCount;
+    int ParallelCount = 0;
-    int RemoteCount;
+    int RemoteCount = 0;
-    int RedistributeCount;
+    int RedistributeCount = 0;
-    int IntervalCount;
+    int IntervalCount = 0;
-    int TotalScoreComm;
+    int TotalScoreComm = 0;
-    int TotalScorePar;
+    int TotalScorePar = 0;
-    int TotalScoreDist;
+    int TotalScoreDist = 0;
 
-    std::string to_string()
+    nlohmann::json toJson()
     {
-        std::string res = "";
+        nlohmann::json stat;
-        res += std::to_string(ParallelCount) + "|";
-        res += std::to_string(RemoteCount) + "|";
-        res += std::to_string(RedistributeCount) + "|";
-        res += std::to_string(IntervalCount) + "|";
 
-        res += std::to_string(ParallelStat.RemoteCount) + "|";
+        stat["ParallelCount"] = ParallelCount;
-        res += std::to_string(ParallelStat.ShadowCount) + "|";
+        stat["RemoteCount"] = RemoteCount;
-        res += std::to_string(ParallelStat.ReductionCount) + "|";
+        stat["RedistributeCount"] = RedistributeCount;
-        res += std::to_string(ParallelStat.AcrossCount);
+        stat["IntervalCount"] = IntervalCount;
 
-        return res;
+        stat["PS_RemoteCount"] = ParallelStat.RemoteCount;
+        stat["PS_ShadowCount"] = ParallelStat.ShadowCount;
+        stat["PS_ReductionCount"] = ParallelStat.ReductionCount;
+        stat["PS_AcrossCount"] = ParallelStat.AcrossCount;
+
+        //TODO: need to improve
+        //                    (summed.TotalScoreComm != 0 ? 1.0 / summed.TotalScoreComm : 0.0) + (double)summed.TotalScorePar * 1000 + (countOfDist == 0 ? -5000 : countOfDist);
+        stat["TotalScore"] = -1 * (ParallelStat.RemoteCount + ParallelStat.ShadowCount + ParallelStat.ReductionCount + ParallelStat.AcrossCount);
+        return stat;
     }
 };
 

src/ProjectManipulation/ParseFiles.cpp

@@ -387,7 +387,50 @@ static string shiftLines(const string &in, const map<string, const FileInfo*> &m
     return newStr;
 }
 
-static int dumpErrors(const vector<FileInfo>& listOfProject, const vector<string>& errors)
+static void addMessage(const string& in, const map<string, const FileInfo*>& mapOfFiles,
+                       const FileInfo* currF, map<string, vector<Messages>>& messages, typeMessage type)
+{
+    int byNum = 0;
+
+    auto it = in.find("on line ");
+    if (it != string::npos)
+        it += strlen("on line ");
+
+    int line = 0;
+    sscanf(in.c_str() + it, "%d", &line);
+
+    auto it1 = in.find("of", it + 1);
+    if (it1 == string::npos)
+        return;
+    it1 += 3;
+
+    string fileN = in.substr(it1, in.find(':', it1) - it1);
+    auto itF = mapOfFiles.find(fileN);
+    if (itF != mapOfFiles.end() && itF->second != currF)
+    {
+        byNum = itF->second->includesAdded;
+        if (byNum != 0)
+        {
+            if (line - byNum <= 0)
+            {
+                //return in;
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+            }
+            else
+                line -= byNum;
+        }
+    }
+    
+    const string newStr = in.substr(0, it) + std::to_string(line) + in.substr(in.find(' ', it + 1));
+
+    wstring messageE, messageR;
+    __spf_printToLongBuf(messageE, L"%s", to_wstring(newStr).c_str());
+    __spf_printToLongBuf(messageR, L"%s", to_wstring(newStr).c_str());
+
+    messages[fileN].push_back(Messages(type, line, messageR, messageE, 6000));
+}
+
+static int dumpErrors(const vector<FileInfo>& listOfProject, const vector<string>& errors, map<string, vector<Messages>>& messages)
 {
     int errorsCount = 0;
     map<string, const FileInfo*> mapOfFiles;
@@ -420,9 +463,13 @@ static int dumpErrors(const vector<FileInfo>& listOfProject, const vector<string
         for (auto& elem : splited)
         {
             if (elem.find("Warning 308") != string::npos)
+            {
+                addMessage(elem, mapOfFiles, &file, messages, WARR);
                 outS += shiftLines(elem, mapOfFiles, &file) + "\n";
+            }
             else if (elem.find("Error") != string::npos)
             {
+                addMessage(elem, mapOfFiles, &file, messages, ERROR);
                 errS += shiftLines(elem, mapOfFiles, &file) + "\n";
                 errorsCount++;
             }
@@ -652,7 +699,7 @@ static int parseFiles(vector<string>& errors, vector<FileInfo>& listOfProject, v
     return rethrow;
 }
 
-int parseFiles(const char* proj, vector<string>& filesCompilationOrder, int parseForInlining)
+int parseFiles(const char* proj, vector<string>& filesCompilationOrder, int parseForInlining, map<string, vector<Messages>>& messages)
 {
     FILE* list = fopen(proj, "r");
     if (!list)
@@ -723,7 +770,7 @@ int parseFiles(const char* proj, vector<string>& filesCompilationOrder, int pars
     vector<string> errors;
 
     int rethrow = parseFiles(errors, listOfProject, filesCompilationOrder, parseForInlining);
-    int errCount = dumpErrors(listOfProject, errors);
+    int errCount = dumpErrors(listOfProject, errors, messages);
 
     if (rethrow != 0)
         throw rethrow;
@@ -732,6 +779,8 @@ int parseFiles(const char* proj, vector<string>& filesCompilationOrder, int pars
 
 void parseFiles(int argc, char** argv)
 {
+    map<string, vector<Messages>> messages;
+
     bool isInline = false;
     auto result = splitCommandLineForParse(argv, argc, isInline);
     if (result.second.size() == 0)

src/ProjectManipulation/ParseFiles.h

@@ -3,5 +3,5 @@
 #include <vector>
 #include <string>
 
-int parseFiles(const char* proj, std::vector<std::string>& filesCompilationOrder, int parseForInlining);
+int parseFiles(const char* proj, std::vector<std::string>& filesCompilationOrder, int parseForInlining, std::map<std::string, std::vector<Messages>>& messages);
 void parseFiles(int argc, char** argv);

src/Sapfor.cpp

@@ -64,6 +64,7 @@
 #include "DvmhRegions/LoopChecker.h"
 #include "DvmhRegions/ReadWriteAnalyzer.h"
 #include "Utils/utils.h"
+#include "Utils/json.hpp"
 #include "Distribution/Array.h"
 
 #include "VisualizerCalls/get_information.h"
@@ -107,6 +108,7 @@ using namespace std;
 using std::chrono::high_resolution_clock;
 using std::chrono::duration_cast;
 using std::chrono::milliseconds;
+using json = nlohmann::json;
 
 int PASSES_DONE[EMPTY_PASS];
 bool PASSES_DONE_INIT = false;
@@ -331,7 +333,8 @@ static string unparseProjectIfNeed(SgFile* file, const int curr_regime, const bo
             unparseToBuf = removeIncludeStatsAndUnparse(file, file_name, fout_name.c_str(), allIncludeFiles, out_free_form == 1, moduleUsesByFile, 
                                                         moduleDecls, getObjectForFileFromMap(file_name, exctactedModuleStats), toString, false, true);
             auto itI = filesToInclude.find(file_name);
-            for (auto& [_, incl] : itI->second)
+            for (auto& [_, incls] : itI->second)
+                for (auto& incl : incls)
                     if (allIncludeFiles.find(incl) != allIncludeFiles.end())
                         allIncludeFiles.erase(incl);
         }
@@ -827,7 +830,7 @@ static bool runAnalysis(SgProject &project, const int curr_regime, const bool ne
         {
             auto fileIt = includeDependencies.find(file_name);
             if (fileIt == includeDependencies.end())
-                fileIt = includeDependencies.insert(fileIt, make_pair(file_name, vector<pair<int, string>>()));
+                fileIt = includeDependencies.insert(fileIt, make_pair(file_name, map<int, set<string>>()));
 
             set<string> modFiles;
             for (auto& elem : moduleDecls)
@@ -844,7 +847,7 @@ static bool runAnalysis(SgProject &project, const int curr_regime, const bool ne
                     {
                         if (lastFromFile == NULL)
                             printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
-                        fileIt->second.push_back(make_pair(lastFromFile->lineNumber(), st->fileName()));
+                        fileIt->second[lastFromFile->lineNumber()].insert(st->fileName());
                     }
                     else
                         lastFromFile = st;
@@ -1603,7 +1606,9 @@ static bool runAnalysis(SgProject &project, const int curr_regime, const bool ne
                     set<string> includedToThisFile;
                     if (itDep != includeDependencies.end())
                     {
-                        for (auto& [_, incl] : itDep->second)
+                        for (auto& [_, incls] : itDep->second)
+                        {
+                            for (auto& incl : incls)
                             {
                                 auto comm = commentsToInclude.find(incl);
                                 if (comm != commentsToInclude.end())
@@ -1611,6 +1616,7 @@ static bool runAnalysis(SgProject &project, const int curr_regime, const bool ne
                                         includedToThisFile.insert(allComm.second.begin(), allComm.second.end());
                             }
                         }
+                    }
 
                     for (int z = 0; z < parallelRegions.size(); ++z)
                     {
@@ -2042,7 +2048,7 @@ static void findFunctionsToInclude(bool needToAddErrors)
                 SPF_messages[byFile.first].push_back(message);
             else
             {
-                if (message.type != ERROR)
+                if (message.getType() != ERROR)
                     SPF_messages[byFile.first].push_back(message);
                 else
                     lastErrors[byFile.first].push_back(message);
@@ -2217,8 +2223,7 @@ void runPass(const int curr_regime, const char *proj_name, const char *folderNam
                 runPass(GROUP_ACTUAL_AND_REMOTE, proj_name, folderName);
 
             runAnalysis(*project, CALCULATE_STATS_SCHEME, false);
-            for (auto& elem : allPredictorStats)
+
-                __spf_print(1, "  stat for file %s: %s\n", elem.first.c_str(), elem.second.to_string().c_str());
             //TODO: need to rewrite this to new algo 
             /*if (!folderName && !consoleMode || predictOn)
                 runAnalysis(*project, PREDICT_SCHEME, false); */
@@ -2386,7 +2391,7 @@ void runPass(const int curr_regime, const char *proj_name, const char *folderNam
         break;
     case PARSE_FILES:
         {
-            int err = parseFiles(proj_name, filesCompilationOrder, parseForInlining);
+            int err = parseFiles(proj_name, filesCompilationOrder, parseForInlining, SPF_messages);
             if (err != 0)
                 throw err;
         }
@@ -2655,26 +2660,28 @@ int main(int argc, char **argv)
     {
         printStackTrace();
         printf("exception occurred\n");
+        
+        json byFileArray = json::array();
         for (auto& byFile : SPF_messages)
         {
+            json inFile;
+            inFile["file"] = byFile.first;
+
+            json messages = json::array();
             for (auto& message : byFile.second)
             {
-                string toPrint = "";
+                message.print(byFile.first);
-                for (int z = 0; z < message.engMessage.size(); ++z)
+                messages.push_back(message.toJson());
-                    toPrint += message.engMessage[z];
+            }
-                string type;
+            inFile["messages"] = messages;
-                if (message.type == WARR)
+            byFileArray.push_back(inFile);
-                    type = "WARR";
+        }
-                else if (message.type == ERROR)
+        json allMessages;
-                    type = "ERROR";
+        allMessages["allMessage"] = byFileArray;
-                else if (message.type == NOTE)
-                    type = "NOTE";
-                else 
-                    type = "UNKN";
 
-                printf("%s - [#%d: %s: line %d]: %s\n", type.c_str(), message.group, byFile.first.c_str(), message.line, toPrint.c_str());
+        FILE* outF = fopen("dump_messages.json", "w");
-            }
+        fprintf(outF, "%s", allMessages.dump().c_str());
-        }
+        fclose(outF);        
     }
 
     deleteAllAllocatedData(withDel);

src/SapforData.h

@@ -85,7 +85,7 @@ std::map<std::string, std::map<int, std::set<std::string>>> commentsToInclude;
 //
 
 //for INSERT_INCLUDES
-std::map<std::string, std::vector<std::pair<int, std::string>>> filesToInclude; // file -> includes [nearest line, include]
+std::map<std::string, std::map<int, std::set<std::string>>> filesToInclude; // file -> includes [nearest line, include]
 //
 
 //for PASSES DEPENDENSIES
@@ -96,7 +96,7 @@ std::set<passes> passesIgnoreStateDone;
 //for files info
 std::map<std::string, int> lineInfo; // file -> lines count
 std::map<std::string, std::pair<std::set<int>, std::set<int>>> dirsInfo; // file -> dirs <lines SPF, lines DVM> count
-std::map<std::string, std::vector<std::pair<int, std::string>>> includeDependencies; // file -> includes [nearest line, include]
+std::map<std::string, std::map<int, std::set<std::string>>> includeDependencies; // file -> includes [nearest line, include]
 std::vector<std::string> filesCompilationOrder; // order of files for unite to one file
 std::map<std::string, std::map<SgStatement*, std::vector<SgStatement*>>> exctactedModuleStats; // file -> hided excluded modules
 //

src/Utils/errors.h

@@ -1,5 +1,7 @@
 #pragma once
 #include <string>
+#include "json.hpp"
+
 #ifdef __SPF
 #include "dvm.h"
 #endif
@@ -143,6 +145,9 @@ enum typeMessage { WARR, ERROR, NOTE };
 //   06 "%s clause can be used only once."
 //   07 "Variable '%s' can't be used in FILES and EXCEPT clauses at the same time."
 
+// 6000 PARSER GROUP
+//
+
 extern int langOfMessages;
 struct Messages
 {
@@ -164,18 +169,41 @@ public:
             engMessage.erase(engMessage.begin() + engMessage.size() - 1);
     }
 
-    std::wstring toString() const
+    nlohmann::json toJson() const 
     {
-        std::wstring retVal = L"|";
+        nlohmann::json resVal;
-        retVal += std::to_wstring((int)type) + L" ";
+
-        retVal += std::to_wstring(line) + L" ";
+        resVal["line"] = line;
-        retVal += std::to_wstring(group);
+        resVal["group"] = group;
-        retVal += L"|" + value;
+        resVal["value"] = std::string(value.begin(), value.end());
-        return retVal; 
+        resVal["type"] = (int)type;
+        return resVal;
     }
 
-    std::string getString() const { return std::string(engMessage.begin(), engMessage.end()); }
+    typeMessage getType() const { return type; }
-public:
+    int getLine() const { return line; }
+
+    void print(const std::string& file) const 
+    {
+        std::string toPrint = "";
+        for (int z = 0; z < engMessage.size(); ++z)
+            toPrint += engMessage[z];
+
+        std::string typeStr;
+        if (type == WARR)
+            typeStr = "WARR";
+        else if (type == ERROR)
+            typeStr = "ERROR";
+        else if (type == NOTE)
+            typeStr = "NOTE";
+        else
+            typeStr = "UNKN";
+
+        printf("%s - [#%d: %s: line %d]: %s\n", typeStr.c_str(), group, file.c_str(), line, toPrint.c_str());
+    }
+
+    auto getUniqKey() const { return std::make_tuple(type, group, line, value); }
+private:
     typeMessage type;
     int group;
     int line;

src/Utils/module_utils.cpp

@@ -343,9 +343,9 @@ const set<SgSymbol*>& getModuleSymbols(SgStatement *func)
     return symbs;
 }
 
-SgSymbol* getNameInLocation(SgStatement* func, const string& varName, const string& locName)
+static void findSymbol(SgStatement* func, const string& varName, const string& locName,
+                       map<string, SgSymbol*>& altNames)
 {
-    map<string, SgSymbol*> altNames;
     for (const auto& s : getModuleSymbols(func))
     {
         SgSymbol* orig = OriginalSymbol(s);
@@ -353,11 +353,22 @@ SgSymbol* getNameInLocation(SgStatement* func, const string& varName, const stri
         if (orig->identifier() == varName && orig->scope()->symbol()->identifier() == locName)
             altNames[s->identifier()] = s;
     }
+}
+
+SgSymbol* getNameInLocation(SgStatement* func, const string& varName, const string& locName)
+{
+    const string clearName = correctSymbolModuleName(varName);
+
+    map<string, SgSymbol*> altNames;
+    findSymbol(func, varName, locName, altNames);
+    
+    if (altNames.size() == 0 && clearName != varName)
+        findSymbol(func, clearName, locName, altNames);
 
     if (altNames.size() > 0)
         return altNames.begin()->second;
     else {
-        __spf_print(1, "%s %s %s\n", func->symbol()->identifier(), varName.c_str(), locName.c_str());
+        __spf_print(1, "%s (%s %s) %s\n", func->symbol()->identifier(), clearName.c_str(), varName.c_str(), locName.c_str());
         printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
     }
 
@@ -393,6 +404,15 @@ SgSymbol* getNameInLocation(SgSymbol* curr, SgStatement* location)
     return returnVal;
 }
 
+string correctSymbolModuleName(const string& origFull)
+{
+    auto it = origFull.find("::");
+    if (it == string::npos)
+        return origFull;
+    else
+        return origFull.substr(it + 2);
+}
+
 namespace Distribution
 {
     const string Array::GetNameInLocation(void* location_p) const

src/Utils/module_utils.h

@@ -1,5 +1,6 @@
 #pragma once
 
+std::string correctSymbolModuleName(const std::string& origFull);
 const std::set<SgSymbol*>& getModuleSymbols(SgStatement* func);
 void getModulesAndFunctions(SgFile* file, std::vector<SgStatement*>& modulesAndFunctions);
 void findModulesInFile(SgFile* file, std::vector<SgStatement*>& modules);

src/Utils/utils.cpp

@@ -30,6 +30,7 @@
 #include "../Distribution/Arrays.h"
 #include "../DynamicAnalysis/gcov_info.h"
 #include "../ParallelizationRegions/ParRegions.h"
+#include "json.hpp"
 
 #if __SPF
 #include "acc_analyzer.h"
@@ -42,6 +43,7 @@ using std::set;
 using std::vector;
 using std::string;
 using std::wstring;
+using json = nlohmann::json;
 
 #if __cplusplus >= 201703L
 #include <filesystem>
@@ -427,7 +429,7 @@ static map<string, vector<Messages>> removeCopies(map<string, vector<Messages>>
         map<tuple<typeMessage, int, int, wstring>, const Messages*> uniq;
         for (auto& message : byFile.second)
         {
-            auto key = make_tuple(message.type, message.group, message.line, message.value);
+            auto key = message.getUniqKey();
             /*string tmp = "";
             for (auto& s : message.toString())
                 tmp += (char)s;
@@ -460,7 +462,7 @@ static void convertGlobalMessagesBuffer(short *&result, int *&resultSize)
         bool waschanged = false;
         for (auto &message : byFile.second)
         {
-            if (message.line > 0)
+            if (message.getLine() > 0)
                 newVal.push_back(message);
             else
                 waschanged = true;
@@ -470,22 +472,30 @@ static void convertGlobalMessagesBuffer(short *&result, int *&resultSize)
             byFile.second = newVal;
     }
 
-    wstring val = L"";
+    json allMessages = json::array();
-    val += std::to_wstring(copySPF_messages.size());
+    for (auto& byFile : copySPF_messages)
-    for (auto it = copySPF_messages.begin(); it != copySPF_messages.end(); ++it)
     {
-        val += L"|" + to_wstring(it->first.c_str()) + L"|" + std::to_wstring(it->second.size());
+        json inFile;
-        for (int k = 0; k < it->second.size(); ++k)
+        inFile["file"] = byFile.first;
-            val += it->second[k].toString();
+
+        json array = json::array();
+        for (auto& message : byFile.second)
+        {
+            json msg = message.toJson();
+            array.push_back(msg);
+        }
+        inFile["messages"] = array;
+        allMessages.push_back(inFile);
     }
 
-    const unsigned len = (unsigned)val.size();
+    json all;
-    result = new short[len + 1];
+    all["allMessages"] = allMessages;
-    allocated.insert(result);
 
-    result[len] = '\0';
+    const string str = all.dump();    
-    for (unsigned i = 0; i < len; ++i)
+    const unsigned len = (unsigned)str.size();
-        result[i] = val[i];
+
+    copyStringToShort(result, str);
+    allocated.insert(result);
 
     resultSize = new int[1];
     resultSize[0] = (int)len;
@@ -1718,3 +1728,15 @@ set<string> fillDistributedArrays(const DataDirective& dataDirectives,
         distrArrays.insert(shortName ? elem->GetShortName() : elem->GetName());
     return distrArrays;
 }
+
+void copyStringToShort(short*& result, const string& resVal, bool withEnd)
+{
+    result = new short[resVal.size() + 1];
+    allocated.insert(result);
+
+    for (int i = 0; i < resVal.size(); ++i)
+        result[i] = resVal[i];
+
+    if (withEnd)
+        result[resVal.size()] = (short)'\0';
+}

src/Utils/utils.h

@@ -96,3 +96,5 @@ std::vector<std::string> splitAndArgvCreate(const std::string& options);
 
 std::set<DIST::Array*> fillDistributedArraysD(const DataDirective& dataDirectives, const std::map<DIST::Array*, std::tuple<int, std::string, std::string>>& tableOfUniqNamesByArray, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls, bool onlyCommon = false);
 std::set<std::string> fillDistributedArrays(const DataDirective& dataDirectives, const std::map<DIST::Array*, std::tuple<int, std::string, std::string>>& tableOfUniqNamesByArray, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls, bool onlyCommon = false, bool shortName = false);
+
+void copyStringToShort(short*& result, const std::string& resVal, bool withEnd = true);

src/Utils/version.h

@@ -1,3 +1,4 @@
 #pragma once
 
-#define VERSION_SPF "2417"
+#define VERSION_SPF "2422"
+

src/VisualizerCalls/get_information.cpp

@@ -218,18 +218,6 @@ static bool tryOpenProjectFile(const char *project)
     return ret;
 }
 
-static void copyStringToShort(short *&result, const string &resVal, bool withEnd = true)
-{
-    result = new short[resVal.size() + 1];
-    allocated.insert(result);
-
-    for (int i = 0; i < resVal.size(); ++i)
-        result[i] = resVal[i];
-
-    if (withEnd)
-        result[resVal.size()] = (short)'\0';
-}
-
 volatile int passDone = 0;
 static volatile int rethrow = 0;
 static void runPassesLoop(const vector<passes> &passesToRun, const char *prName, const char *folderNameChar)
@@ -465,14 +453,15 @@ int SPF_ParseFilesWithOrder(void*& context, int winHandler, short* options, shor
     {
         runPassesForVisualizer(projName, { PARSE_FILES });
 
-        string resVal = "";
+        json filesArray = json::array();
-        for (auto& elem : filesCompilationOrder)
+        
-        {
+        for (auto& file : filesCompilationOrder)
-            if (resVal == "")
+            filesArray.push_back(file);            
-                resVal += elem;
+        
-            else
+        json filesOrder;
-                resVal += "|" + elem;
+        filesOrder["allFiles"] = filesArray;
-        }
+
+        string resVal = filesOrder.dump();
         copyStringToShort(result, resVal);
         retSize = 0;
     }
@@ -766,15 +755,17 @@ int SPF_GetArrayDistribution(void*& context, int winHandler, short *options, sho
         else
             printInternalError(convertFileName(__FILE__).c_str(), __LINE__);            
 
-        string resVal = "";
+        json regions = json::array();
-        resVal += to_string(parallelRegions.size());
+        for (auto& reg : parallelRegions)
-        for (int i = 0; i < parallelRegions.size(); ++i)
+        {
-            resVal += parallelRegions[i]->toString();
+            json currReg = reg->toJson();
+            regions.push_back(currReg);
+        }
 
-        //__spf_print(1, "==============\n");
+        json allRegions;
-        //__spf_print(1, "%s\n", resVal.c_str());
+        allRegions["allRegions"] = regions;
-        //__spf_print(1, "==============\n");
 
+        string resVal = allRegions.dump();
         copyStringToShort(result, resVal);
         retSize = (int)resVal.size() + 1;
     }
@@ -1063,7 +1054,6 @@ int SPF_CreateParallelVariant(void*& context, int winHandler, short *options, sh
             printf("SAPFOR: set all info done\n");
         runPassesForVisualizer(projName, { INSERT_PARALLEL_DIRS }, folderName);
 
-        string predictRes = "";
         PredictorStats summed;
         for (auto &predFile : allPredictorStats)
         {
@@ -1080,43 +1070,10 @@ int SPF_CreateParallelVariant(void*& context, int winHandler, short *options, sh
             summed.TotalScoreDist += predFile.second.TotalScoreDist;
             summed.TotalScorePar += predFile.second.TotalScorePar;
         }
-        predictRes += summed.to_string();
-        //predictRes += "|" + to_string((summed.TotalScoreComm != 0 ? 1.0 / summed.TotalScoreComm : 0.0 )+ (double)summed.TotalScorePar * 1000 + (countOfDist == 0 ? -5000 : countOfDist));
-        if (countOfDist == 0)
-            predictRes += "|x";
-        else
-            predictRes += "|" + to_string(-1 * (summed.ParallelStat.AcrossCount + summed.ParallelStat.RemoteCount + summed.RedistributeCount + summed.RemoteCount));
-        //predictRes += "|0";
-
-        //TODO: need to rewrite to new algo
-        /*if (folderName == NULL)
-        {
-            SpfInterval *mainIterval = getMainInterval(project, intervals);
-            const int idxBest = mainIterval->getBestTimeIdx();
-            double speedUpBest = 1;
-            int procCount = 1;
-            string topo = "";
-            if (idxBest != -1 && mainIterval->exec_time != 0)
-            {
-                speedUpBest = mainIterval->exec_time / mainIterval->predictedTimes[idxBest];
-                topo += "[";
-                for (int z = 0; z < topologies[idxBest].size(); ++z)
-                {
-                    topo += to_string(topologies[idxBest][z]);
-                    procCount *= topologies[idxBest][z];
-                    if (z != topologies[idxBest].size() - 1)
-                        topo += "x";
-                }
-                topo += "]";
-            }
-            char buf[256];
-            sprintf(buf, "%.2f", speedUpBest / procCount * 100.0);
-            predictRes += "|" + string(buf) + topo;
-        }
-        else
-            predictRes += "|0";*/
 
+        string predictRes = summed.toJson().dump();                
         copyStringToShort(predictorStats, predictRes);
+
         __spf_print(1, "  statistic to send: %s\n", predictRes.c_str());
         retSize = (int)predictRes.size();
     }
@@ -1156,8 +1113,6 @@ int SPF_GetArrayLinks(void*& context, int winHandler, short *options, short *pro
     {
         runPassesForVisualizer(projName, { CALL_GRAPH2 } );
 
-        string resVal = "";
-        
         map<DIST::Array*, set<DIST::Array*>> linkedArrays;
         for (auto& inMap : arrayLinksByFuncCalls)
         {
@@ -1171,17 +1126,24 @@ int SPF_GetArrayLinks(void*& context, int winHandler, short *options, short *pro
                             linkedArrays[ref].insert(toAdd);
         }
 
-
+        json allLinks = json::array();
         for (auto& array : linkedArrays)
         {
-            resVal += to_string(array.first->GetId()) + "|" + to_string(array.second.size()) + "|";
+            json currLink;
+            currLink["id"] = array.first->GetId();
+
+            json links = json::array();
             for (auto& link : array.second)
-                resVal += to_string(link->GetId()) + "|";
+                links.push_back(link->GetId());
+            currLink["links"] = links;
+
+            allLinks.push_back(currLink);
         }
 
-        if (resVal[resVal.size() - 1] == '|')
+        json links;
-            resVal.erase(resVal.size() - 1);
+        links["allLinks"] = allLinks;
 
+        string resVal = links.dump();
         copyStringToShort(result, resVal);
         retSize = (int)resVal.size() + 1;
     }
@@ -1308,7 +1270,7 @@ int SPF_GetIntrinsics(void*& context, short *&result)
     return (int)resVal.size() + 1;
 }
 
-extern map<string, vector<pair<int, string>>> includeDependencies;
+extern map<string, map<int, set<string>>> includeDependencies;
 int SPF_GetIncludeDependencies(void*& context, int winHandler, short *options, short *projName, short *&result, short*& output, int*& outputSize,
                                short*& outputMessage, int*& outputMessageSize)
 {
@@ -1328,13 +1290,16 @@ int SPF_GetIncludeDependencies(void*& context, int winHandler, short *options, s
             includes["file"] = deps.first;
 
             json array = json::array();
-            for (const auto& [line, incl] : deps.second)
+            for (const auto& [line, incls] : deps.second)
+            {
+                for (auto& incl : incls)
                 {
                     json elem;
                     elem["line"] = line;
                     elem["dependencyFileName"] = incl;
                     array.push_back(elem);
                 }
+            }
             includes["includes"] = array;
             inc_array.push_back(includes);
         }
@@ -2192,7 +2157,7 @@ int SPF_InlineProcedures(void*& context, int winHandler, short* options, short*
 }
 
 
-extern map<string, vector<pair<int, string>>> filesToInclude;
+extern map<string, map<int, set<string>>> filesToInclude;
 int SPF_InsertIncludesPass(void*& context, int winHandler, short *options, short *projName, short *folderName, char *visFilesToInclude,
                            short *&output, int *&outputSize, short *&outputMessage, int *&outputMessageSize)
 {
@@ -2225,9 +2190,8 @@ int SPF_InsertIncludesPass(void*& context, int winHandler, short *options, short
             if (sscanf(splited[k].c_str(), "%d", &line) == -1)
                 return -5;
 
-            auto pair = make_pair(line, splited[k + 1]);
+            filesToInclude[file][line].insert(splited[k + 1]);
-            filesToInclude[file].push_back(pair);
+            __spf_print(1, "    include = [%d %s]\n", line, splited[k + 1].c_str());
-            __spf_print(1, "    include = [%d %s]\n", pair.first, pair.second.c_str());
         }
         i += 2 * num;
     }