get def use for array

dvm/fdvm/trunk/Sage/h/tag

@@ -236,6 +236,7 @@
 #define DVM_EXIT_INTERVAL_DIR	 639    /* DVM-F */
 #define DVM_TEMPLATE_CREATE_DIR	 640    /* DVM-F */ 
 #define DVM_TEMPLATE_DELETE_DIR	 641    /* DVM-F */
+#define PRIVATE_AR_DECL          642    /* DVM-F */
     
 /***************** variant tags for low level nodes ********************/
 

dvm/fdvm/trunk/Sage/h/tag.h

@@ -238,7 +238,8 @@ script using "tag".  Run make tag.h to regenerate this file */
   tag [ DVM_EXIT_INTERVAL_DIR ] = "DVM_EXIT_INTERVAL_DIR";
   tag [ DVM_TEMPLATE_CREATE_DIR ] = "DVM_TEMPLATE_CREATE_DIR";
   tag [ DVM_TEMPLATE_DELETE_DIR ] = "DVM_TEMPLATE_DELETE_DIR";
-    
+  tag [ PRIVATE_AR_DECL ] = "PRIVATE_AR_DECL";
+  
 /***************** variant tags for low level nodes ********************/
 
   tag [ INT_VAL ] = "INT_VAL";

dvm/fdvm/trunk/Sage/lib/include/unparseC++.def

@@ -139,6 +139,8 @@ DEFNODECODE(CONT_STAT,          "%CMNT%PUTTABcontinue;%NL",
 's',0,BIFNODE)
 DEFNODECODE(VAR_DECL,           "%CMNT%SETFLAG(VARDECL)%IF (%CHECKFLAG(ENUM) == %NULL)%IF (%CHECKFLAG(CLASSDECL) != %NULL)%PROTECTION%ENDIF%PUTTAB%DECLSPEC%TYPE %ENDIF%LL1%IF (%CHECKFLAG(ENUM) == %NULL);%ENDIF%UNSETFLAG(VARDECL)%NL",
 's',0,BIFNODE)
+DEFNODECODE(PRIVATE_AR_DECL,    "%CMNT%PUTTABPrivateArray<%LL1,%LL2> %LL3;%NL",
+'s',0,BIFNODE)
 DEFNODECODE(PARAM_DECL,         "%ERROR",
 's',0,BIFNODE)
 DEFNODECODE(COMM_STAT,          "%ERROR",

dvm/fdvm/trunk/fdvm/acc_across.cpp

@@ -633,7 +633,7 @@ ArgsForKernel *Create_C_Adapter_Function_Across(SgSymbol *sadapter)
     // clear information
     allRegNames.clear();
 
-    SgStatement *st_hedr, *st_end, *first_exec, *stmt;
+    SgStatement *st_hedr=NULL, *st_end, *first_exec, *stmt;
     vector<SgStatement*> cuda_kernel;
     SgExpression *fe, *ae, *el, *arg_list;
     SgType *typ;
@@ -698,8 +698,9 @@ ArgsForKernel *Create_C_Adapter_Function_Across(SgSymbol *sadapter)
                 kernel_symbNew += "_long";
             else if (rtTypes[t] == rt_LLONG)
                 kernel_symbNew += "_llong";
-
+           
             cuda_kernel[t] = CreateLoopKernelAcross(new SgSymbol(FUNCTION_NAME, kernel_symbNew.c_str(), *C_VoidType(), *block_C), &retValueForKernel[t], indexTypeInKernel(rtTypes[t]));
+            
             if (options.isOn(RTC))
             {
                 acc_call_list = ACC_RTC_ExpandCallList(acc_call_list);
@@ -723,7 +724,7 @@ ArgsForKernel *Create_C_Adapter_Function_Across(SgSymbol *sadapter)
         // if only type ~ 1 across symb
         bool ifOne = true;
         for (size_t i = 0; i < allVariants.size(); ++i)
-        {
+        {    
             if (allVariants[i].acrossV != 1)
                 ifOne = false;
         }
@@ -734,7 +735,7 @@ ArgsForKernel *Create_C_Adapter_Function_Across(SgSymbol *sadapter)
             dontGenConvertXY = false;
 
         for (size_t i = 0; i < allVariants.size(); ++i)
-        {
+        {    
 #if debugMode
                 printf("%d case\n", allVariants[i].type);
 #endif
@@ -758,7 +759,7 @@ ArgsForKernel *Create_C_Adapter_Function_Across(SgSymbol *sadapter)
                     kernel_symb += "_long";
                 else if (rtTypes[k] == rt_LLONG)
                     kernel_symb += "_llong";
-
+        
                 if (tmp.acrossV == 1 && tmp.type == 1)
                 {
                     if (k == 0) // create CUDA handler once
@@ -788,7 +789,7 @@ ArgsForKernel *Create_C_Adapter_Function_Across(SgSymbol *sadapter)
                         }
                     }
                 }
-
+               
                 if (newVars.size() != 0)
                 {
                     correctPrivateList(RESTORE);
@@ -839,7 +840,7 @@ ArgsForKernel *Create_C_Adapter_Function_Across(SgSymbol *sadapter)
         first_exec = st_end;
         mywarn("start: create  dummy argument list ");
 
-        // create  dummy argument list: loop_ref, <dvm-array-headers>, <uses>
+        // create  dummy argument list: loop_ref, <dvm-array-headers>, <uses> ,<private-array-shapes>
         typ = C_PointerType(C_Derived_Type(s_DvmhLoopRef));
         s_loop_ref = new SgSymbol(VARIABLE_NAME, "loop_ref", *typ, *st_hedr);
         argsForVariantFunction.push_back(s_loop_ref);
@@ -864,7 +865,7 @@ ArgsForKernel *Create_C_Adapter_Function_Across(SgSymbol *sadapter)
             arg_list->setRhs(*new SgExprListExp(*ae));
             arg_list = arg_list->rhs();
         }
-
+        
         for (el = uses_list; el; el = el->rhs())    // <uses>
         {
             s = el->lhs()->symbol();
@@ -879,6 +880,46 @@ ArgsForKernel *Create_C_Adapter_Function_Across(SgSymbol *sadapter)
             arg_list->setRhs(*new SgExprListExp(*ae));
             arg_list = arg_list->rhs();
         }
+     
+        if (options.isOn(C_CUDA))    // <private-array-shapes>
+        {  
+            int idim;
+            SgExpression *elp;
+            SgType *t = C_PointerType(C_DvmType());
+
+            for (elp=private_list; elp; elp = elp->rhs())
+            {
+                s = elp->lhs()->symbol();
+                if (IS_ARRAY(s) && !TestArrayShape(s))     
+                {   
+                    el = NULL;
+                    for (idim = 1; idim<=Rank(s); idim++)
+                    {
+                        sarg = new SgSymbol(VARIABLE_NAME, DimSizeName(s, idim), *t, *st_hedr);
+                        argsForVariantFunction.push_back(sarg);
+                        ae = new SgVarRefExp(sarg);
+                        ae->setType(t);
+                        ae = new SgPointerDerefExp(*ae);
+                        arg_list->setRhs(*new SgExprListExp(*ae));
+                        arg_list = arg_list->rhs();
+
+                    }
+                    el = NULL;
+                    for (idim = 1; idim<=Rank(s); idim++)
+                    {
+                        sarg = new SgSymbol(VARIABLE_NAME, BoundName(s, idim, 1), *t, *st_hedr);
+                        argsForVariantFunction.push_back(sarg);
+                        ae = new SgVarRefExp(sarg);
+                        ae->setType(t);
+                        ae = new SgPointerDerefExp(*ae);
+                        arg_list->setRhs(*new SgExprListExp(*ae));
+                        arg_list = arg_list->rhs();
+                    } 
+                }
+
+            }
+        } 
+
         mywarn("  end: create  dummy argument list ");
 
         mywarn("start: create  IF BLOCK ");
@@ -1112,9 +1153,10 @@ ArgsForKernel *Create_C_Adapter_Function_Across(SgSymbol *sadapter)
 
         mywarn("  end: create  IF BLOCK ");
     }
+  
     if (options.isOn(C_CUDA))   
         RenamingCudaFunctionVariables(st_hedr, s_loop_ref, 0); //(st_hedr, current_symbol->next(), 0);
- 
+
     return NULL;
 }
 
@@ -1127,14 +1169,14 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_OneThread(SgSymbol *sadap
     SgSymbol **reduction_ptr;
     SgSymbol *lowI, *highI, *idxI;
     symb_list *sl;
-    SgStatement *st_hedr, *st_end, *stmt, *first_exec;
-    SgExpression *fe, *ae, *arg_list, *el, *e, *espec, *er;
-    SgSymbol *s_loop_ref, *sarg, *s, *sb, *sg, *sdev, *h_first, *hgpu_first, *base_first, *uses_first, *scalar_first;
+    SgStatement *st_hedr, *st_end, *stmt, *first_exec, *stmt_save;
+    SgExpression *fe, *ae, *arg_list, *el, *e, *espec, *er, *e_all_private_size = NULL;
+    SgSymbol *s_loop_ref, *sarg, *s, *sb, *sg, *sdev, *h_first, *hgpu_first, *base_first, *uses_first, *scalar_first, *private_first=NULL;
     SgSymbol *s_blocks, *s_threads, *s_dev_num, *s_tmp_var, *idxTypeInKernel;
     SgType *typ;
     SgFunctionCallExp *funcCall;
     vector<char*> dvm_array_headers;
-    int ln, num, uses_num, has_red_array, use_device_num, num_of_red_arrays = 0, nbuf = 0;
+    int ln, num, uses_num, has_red_array, use_device_num, num_of_red_arrays = 0, nbuf = 0, lnp = 0;
 
     // init block
     reduction_ptr = NULL;
@@ -1206,8 +1248,58 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_OneThread(SgSymbol *sadap
     }
     uses_num = ln;
 
-    mywarn("  end: create  dummy argument list ");
+    if (options.isOn(C_CUDA))  // <private-array-shapes>
+    {
+        int idim;
+        SgExpression *elp;
+        SgType *t = C_PointerType(C_DvmType());
 
+        for (elp=private_list; elp; elp = elp->rhs())
+        {
+            s = elp->lhs()->symbol();
+            if (IS_ARRAY(s) && !TestArrayShape(s))     
+            {   
+                el = NULL;
+                for (idim = Rank(s); idim; idim--)
+                {
+                    sarg = new SgSymbol(VARIABLE_NAME, DimSizeName(s, idim), *t, *st_hedr);
+                    ae = new SgVarRefExp(sarg);
+                    ae->setType(t);
+                    el = AddElementToList(el, new SgPointerDerefExp(*ae));
+                }
+                arg_list = AddListToList(arg_list, &el->copy());
+                if (!elp->lhs()->attributeValue(0, DIM_SIZES))
+                {
+                    SgExpression **edim = new (SgExpression *);
+                    *edim = el;
+                    elp->lhs()->addAttribute(DIM_SIZES, (void *)edim, sizeof(SgExpression *) ); 
+                }   
+
+                el = NULL;
+                for (idim = Rank(s); idim; idim--)
+                {
+                    sarg = new SgSymbol(VARIABLE_NAME, BoundName(s, idim, 1), *t, *st_hedr);
+                    ae = new SgVarRefExp(sarg);
+                    ae->setType(t);
+                    el = AddElementToList(el, new SgPointerDerefExp(*ae));
+                } 
+                arg_list = AddListToList(arg_list, &el->copy());
+                if (!elp->lhs()->attributeValue(0, L_BOUNDS))
+                {
+                    SgExpression **elb = new (SgExpression *);
+                    *elb = el;
+                    elp->lhs()->addAttribute(L_BOUNDS, (void *)elb, sizeof(SgExpression *) ); 
+                }
+
+                while (arg_list->rhs() != 0)
+                    arg_list = arg_list->rhs();
+            }
+
+        }
+    } 
+
+    mywarn("  end: create  dummy argument list ");
+    // create variable's declarations: <dvm_array_headers>,<dvm_array_bases>,<scalar_device_addr>,<reduction_variables>,<private-arrays>,blocks_info [ or blocksS,idxL,idxH ],stream,blocks,threads
     if (red_list) // reduction section
     {
         mywarn("start: in reduction section ");
@@ -1269,7 +1361,7 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_OneThread(SgSymbol *sadap
                     btype = loc_type->baseType();
                 else
                     btype = loc_type;
-                //!printf("__112\n");
+                
                 SgArrayType *typearray = new SgArrayType(*C_Type(btype));
                 typearray->addRange(*new SgValueExp(loc_el_num));
                 s_loc_var->setType(*typearray);
@@ -1282,7 +1374,6 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_OneThread(SgSymbol *sadap
                 st_hedr->insertStmtAfter(*stmt, *st_hedr);
             }
 
-            //!printf("__113\n");
             /*--- executable statements: register reductions in RTS ---*/
             e = &SgAssignOp(*new SgVarRefExp(s_tmp_var), *new SgValueExp(ln+1));
             stmt = new SgCExpStmt(*e);
@@ -1438,10 +1529,12 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_OneThread(SgSymbol *sadap
     st_end->insertStmtBefore(*stmt, *st_hedr);
     stmt->addComment("// Get bounds");
     mywarn("  end: create assigns");
-
+    stmt_save = stmt;
+    
     stmt = new SgCExpStmt(SgAssignOp(*new SgRecordRefExp(*s_blocks, "x"), *new SgValueExp(1)));
     st_end->insertStmtBefore(*stmt, *st_hedr);
     stmt->addComment("// Start counting");
+    SgStatement *st_where = stmt;
 
     stmt = new SgCExpStmt(SgAssignOp(*new SgRecordRefExp(*s_threads, "x"), *new SgValueExp(1)));
     st_end->insertStmtBefore(*stmt, *st_hedr);
@@ -1495,6 +1588,7 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_OneThread(SgSymbol *sadap
             for (int i = NumberOfCoeffs(sg); i>0; i--)
                 funcCall->addArg(*new SgArrayRefExp(*sg, *new SgValueExp(i)));
         }
+
         if (red_list)
         {
             reduction_operation_list *rsl;
@@ -1520,6 +1614,7 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_OneThread(SgSymbol *sadap
                         funcCall->addArg(*new SgArrayRefExp(*s, *new SgValueExp(i)));
                 }
                 s = s->next();
+
                 if (options.isOn(C_CUDA))
                     funcCall->addArg(*new SgVarRefExp(reduction_ptr[i]));
                 else
@@ -1543,6 +1638,35 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_OneThread(SgSymbol *sadap
             }
         }
 
+    e_all_private_size = sizeOfPrivateArraysInBytes();
+    if (options.isOn(C_CUDA) && e_all_private_size)
+    {
+        for (el=private_list, lnp=0; el; el=el->rhs())
+        {
+             s = el->lhs()->symbol();
+             if (IS_ARRAY(s))
+             {                
+                 sarg = new SgSymbol(VARIABLE_NAME, PointerNameForPrivateArray(s), *C_PointerType(C_VoidType()), *st_hedr);
+                 ae = new SgCastExp(*C_PointerType( C_Type(s->type()->baseType())), *new SgVarRefExp(sarg));
+                 funcCall->addArg(*ae);
+                 if (!lnp)
+                     private_first = sarg;
+                 lnp++;        
+                 if (!TestArrayShape(s))
+                 {   
+                     SgExpression **eatr = (SgExpression **) el->lhs()->attributeValue(0, DIM_SIZES);  
+                     SgExpression *ela;
+                     for (ela = *eatr; ela; ela = ela->rhs())
+                         funcCall->addArg(SgDerefOp(*new SgVarRefExp(ela->lhs()->lhs()->symbol())));  
+                     eatr = (SgExpression **) el->lhs()->attributeValue(0, L_BOUNDS);    
+                     for (ela = *eatr; ela; ela = ela->rhs())
+                         funcCall->addArg(SgDerefOp(*new SgVarRefExp(ela->lhs()->lhs()->symbol()))); 
+                 }
+
+             }
+         }
+    }         
+
         for (int i = 0; i < acrossV + loopV; ++i)
         {
             funcCall->addArg(*new SgArrayRefExp(*lowI, *new SgValueExp(i)));
@@ -1557,7 +1681,7 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_OneThread(SgSymbol *sadap
     if (red_list)
     {
         ln = 0;
-        for (er = red_list; er; er = er->rhs(), ++ln)
+        for (er = red_list, s = red_first; er; er = er->rhs(), ++ln, s=s->next())
         {
             funcCall = new SgFunctionCallExp(*createNewFunctionSymbol("cudaMemcpy"));
             funcCall->addArg(SgAddrOp(*new SgVarRefExp(&(er->lhs()->rhs()->symbol()->copy()))));
@@ -1571,9 +1695,10 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_OneThread(SgSymbol *sadap
             stmt = new SgCExpStmt(*e);
             st_end->insertStmtBefore(*stmt, *st_hedr);
 
-            stmt = new SgCExpStmt(*RedPost(s_loop_ref, s_tmp_var, &(er->lhs()->rhs()->symbol()->copy()), NULL)); // loop_red_post_
-            st_end->insertStmtBefore(*stmt, *st_hedr);
-        }
+            stmt = new SgCExpStmt(*RedPost(s_loop_ref, s_tmp_var, s, NULL)); // loop_red_post_
+            st_end->insertStmtBefore(*stmt, *st_hedr);            
+        }    
+
         ln = 0;
         for (er = red_list; er; er = er->rhs(), ++ln)
         {
@@ -1585,6 +1710,18 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_OneThread(SgSymbol *sadap
                 stmt->addComment("// Free temporary variables");
         }
     }
+    // insert code for big private arrays
+    if (options.isOn(C_CUDA) && e_all_private_size) 
+    {
+        GetMemoryForPrivateArrays(private_first, s_loop_ref, lnp, st_where, st_hedr, new SgValueExp(1));        
+
+   // to dispose private arrays
+        for (s = private_first, ln = 0;  ln < lnp; s = s->next(), ln++)  // private arrays
+        {
+            stmt = new SgCExpStmt(*DisposePrivateArray(s_loop_ref, s));
+	    st_end->insertStmtBefore(*stmt, *st_hedr);  
+        }
+    }
     // create args for kernel and return it
     vector<ArgsForKernel> argsKernel(countKernels);
     for (unsigned i = 0; i < countKernels; ++i)
@@ -1594,6 +1731,7 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_OneThread(SgSymbol *sadap
     mywarn("  end Adapter Function");
     if (options.isOn(C_CUDA))
         RenamingCudaFunctionVariables(st_hedr, s_loop_ref, 0);
+    
     return argsKernel;
 }
 
@@ -1602,8 +1740,8 @@ static inline void insertReductionArgs(SgSymbol **reduction_ptr, SgSymbol **redu
                                        SgFunctionCallExp *funcCallKernel, SgSymbol* numBlocks, int &has_red_array)
 {
     reduction_operation_list *rsl;
-    SgSymbol *s;
-    SgExpression *e;
+    SgSymbol *s = NULL;
+    SgExpression *e = NULL;
 
     for (rsl = red_struct_list, s = red_first; rsl; rsl = rsl->next)  //s!=s_blocks_info
     {
@@ -1640,19 +1778,43 @@ static inline void insertReductionArgs(SgSymbol **reduction_ptr, SgSymbol **redu
         else
             funcCallKernel->addArg(*new SgCastExp(*C_PointerType(new SgDescriptType(*SgTypeChar(), BIT_SIGNED)), *new SgVarRefExp(reduction_ptr[i])));
 
-        //TODO!!
         if (rsl->locvar)  //MAXLOC,MINLOC
         {
             for (int k = 0; k < rsl->number; ++k)
                 funcCallKernel->addArg(*new SgArrayRefExp(*reduction_loc_symb[i], *new SgValueExp(k)));
             s = s->next();
-            e = new SgCastExp(*C_PointerType(options.isOn(C_CUDA) ? C_Type(rsl->locvar->type()) : new SgDescriptType(*SgTypeChar(), BIT_SIGNED)), *new SgVarRefExp(s));
+
+            if (options.isOn(C_CUDA))
+                e = new SgCastExp(*C_PointerType(C_Type(rsl->locvar->type())), *new SgVarRefExp(reduction_loc_ptr[i]));
+            else
+                e = new SgCastExp(*C_PointerType(new SgDescriptType(*SgTypeChar(), BIT_SIGNED)), *new SgVarRefExp(s));// TODO it like in C_Cuda
             funcCallKernel->addArg(*e);
             s = s->next();
         }
     }
 }
 
+static void createPrivatePointers(SgSymbol* &private_first, int &lnp, SgStatement* st_hedr, SgExpression* &e_all_private_size)
+{
+    private_first = NULL;
+    if (options.isOn(C_CUDA) && (e_all_private_size=sizeOfPrivateArraysInBytes()))
+    {
+        SgExpression *el, *ae;
+        SgSymbol *sarg;
+       
+        for (el=private_list, lnp=0; el; el=el->rhs())
+        {
+            SgSymbol *s = el->lhs()->symbol();
+            if (IS_ARRAY(s))
+            {                
+                sarg = new SgSymbol(VARIABLE_NAME, PointerNameForPrivateArray(s), *C_PointerType(C_VoidType()), *st_hedr);
+                if (!lnp)
+                    private_first = sarg;
+                lnp++;        
+            }
+        }
+    }         
+} 
 
 static void createArgsForKernelForTwoDeps(SgFunctionCallExp*& funcCallKernel, SgSymbol* kernel_symb, SgExpression* espec, SgSymbol*& sg, SgSymbol* hgpu_first,
                                           SgSymbol*& sb, SgSymbol* base_first, symb_list*& sl, int& ln, int num, SgExpression*& e, SgSymbol** reduction_ptr,
@@ -1660,7 +1822,7 @@ static void createArgsForKernelForTwoDeps(SgFunctionCallExp*& funcCallKernel, Sg
                                           SgSymbol* diag, const int& loopV, SgSymbol** num_elems, const int& acrossV, SgSymbol* acrossBase[16], SgSymbol* loopBase[16],
                                           SgSymbol* idxI, const vector<SageSymbols>& loopAcrossSymb, const vector<SageSymbols>& loopSymb, SgSymbol*& s, SgSymbol* uses_first,
                                           SgSymbol*& sdev, SgSymbol* scalar_first, int uses_num, vector<char*>& dvm_array_headers,
-                                          SgSymbol** addressingParams, SgSymbol** outTypeOfTransformation, SgSymbol* type_of_run, SgSymbol* bIdxs)
+                                          SgSymbol** addressingParams, SgSymbol** outTypeOfTransformation, SgSymbol* type_of_run, SgSymbol* bIdxs, SgSymbol* private_first, int lnp)
 {
 
     funcCallKernel = CallKernel(kernel_symb, espec);
@@ -1727,6 +1889,32 @@ static void createArgsForKernelForTwoDeps(SgFunctionCallExp*& funcCallKernel, Sg
             sdev = sdev->next();
         }
     }
+           
+    if (options.isOn(C_CUDA) && private_first) // there are big private arrays
+    {
+        SgExpression *el, *ae;
+        SgSymbol *sarg, *sp, *s;
+        int ln;
+        for (sp = private_first, el = private_list, ln = 0;  ln < lnp; sp = sp->next(), el = el->rhs(), ln++) 
+        {            
+            while (!IS_ARRAY(el->lhs()->symbol()))
+                el = el->rhs();
+            s = el->lhs()->symbol(); 
+            ae = new SgCastExp(*C_PointerType( C_Type(s->type()->baseType())), *new SgVarRefExp(sp));
+            funcCallKernel->addArg(*ae);
+            if (!TestArrayShape(s))
+            {   
+                SgExpression **eatr = (SgExpression **) el->lhs()->attributeValue(0, DIM_SIZES);  
+                SgExpression *ela;
+                for (ela = *eatr; ela; ela = ela->rhs())
+                    funcCallKernel->addArg(SgDerefOp(*new SgVarRefExp(ela->lhs()->lhs()->symbol())));  
+                eatr = (SgExpression **) el->lhs()->attributeValue(0, L_BOUNDS);    
+                for (ela = *eatr; ela; ela = ela->rhs())
+                    funcCallKernel->addArg(SgDerefOp(*new SgVarRefExp(ela->lhs()->lhs()->symbol()))); 
+            }
+
+        }
+    }         
 
     if (options.isOn(AUTO_TFM))
     {
@@ -1767,14 +1955,14 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
 
     symb_list *sl;
     SgStatement *st_hedr, *st_end, *stmt, *first_exec;
-    SgExpression *fe, *ae, *arg_list, *el, *e, *espec, *ex, *er;
-    SgSymbol *s_loop_ref, *sarg, *s, *sb, *sg, *sdev, *h_first, *hgpu_first, *base_first, *uses_first, *scalar_first;
+    SgExpression *fe, *ae, *arg_list, *el, *e, *espec, *ex, *er, *e_all_private_size = NULL, *e_totalThreads;
+    SgSymbol *s_loop_ref, *sarg, *s, *sb, *sg, *sdev, *h_first, *hgpu_first, *base_first, *uses_first, *scalar_first, *private_first;
     SgSymbol *s_blocks, *s_threads, *s_dev_num, *s_tmp_var, *type_of_run, *s_i = NULL, *s_k = NULL, *s_tmp_var_1;
     SgSymbol *idxTypeInKernel;
     SgType *typ;
     SgFunctionCallExp *funcCall, *funcCallKernel;
     vector<char*> dvm_array_headers;
-    int ln, num, uses_num, has_red_array, use_device_num, num_of_red_arrays, nbuf = 0;
+    int ln, num, uses_num, has_red_array, use_device_num, num_of_red_arrays, nbuf = 0, lnp;
 
     // init block
     lowI = highI = idxI = elem = red_blocks = shared_mem = stream_t = bIdxs = NULL;
@@ -1850,6 +2038,56 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
     }
     uses_num = ln;
 
+    if (options.isOn(C_CUDA))  // <private-array-shapes>
+    {
+        int idim;
+        SgExpression *elp;
+        SgType *t = C_PointerType(C_DvmType());
+
+        for (elp=private_list; elp; elp = elp->rhs())
+        {
+            s = elp->lhs()->symbol();
+            if (IS_ARRAY(s) && !TestArrayShape(s))     
+            {   
+                el = NULL;
+                for (idim = Rank(s); idim; idim--)
+                {
+                    sarg = new SgSymbol(VARIABLE_NAME, DimSizeName(s, idim), *t, *st_hedr);
+                    ae = new SgVarRefExp(sarg);
+                    ae->setType(t);
+                    el = AddElementToList(el, new SgPointerDerefExp(*ae));
+                }
+                arg_list = AddListToList(arg_list, &el->copy());
+                if (!elp->lhs()->attributeValue(0, DIM_SIZES))
+                {
+                    SgExpression **edim = new (SgExpression *);
+                    *edim = el;
+                    elp->lhs()->addAttribute(DIM_SIZES, (void *)edim, sizeof(SgExpression *) ); 
+                }   
+
+                el = NULL;
+                for (idim = Rank(s); idim; idim--)
+                {
+                    sarg = new SgSymbol(VARIABLE_NAME, BoundName(s, idim, 1), *t, *st_hedr);
+                    ae = new SgVarRefExp(sarg);
+                    ae->setType(t);
+                    el = AddElementToList(el, new SgPointerDerefExp(*ae));
+                } 
+                arg_list = AddListToList(arg_list, &el->copy());
+                if (!elp->lhs()->attributeValue(0, L_BOUNDS))
+                {
+                    SgExpression **elb = new (SgExpression *);
+                    *elb = el;
+                    elp->lhs()->addAttribute(L_BOUNDS, (void *)elb, sizeof(SgExpression *) ); 
+                }
+
+                while (arg_list->rhs() != 0)
+                    arg_list = arg_list->rhs();
+            }
+
+        }
+    } 
+
     type_of_run = new SgSymbol(VARIABLE_NAME, TestAndCorrectName("type_of_run"), *LongT, *st_hedr);
     ae = new SgVarRefExp(type_of_run);
     ae->setType(LongT);
@@ -1941,7 +2179,7 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
                     btype = loc_type->baseType();
                 else
                     btype = loc_type;
-                //!printf("__112\n");
+                
                 SgArrayType *typearray = new SgArrayType(*C_Type(btype));
                 typearray->addRange(*new SgValueExp(loc_el_num));
                 s_loc_var->setType(*typearray);
@@ -1955,7 +2193,7 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
                 st_hedr->insertStmtAfter(*stmt, *st_hedr);
             }
 
-            //!printf("__113\n");
+           
             /*--- executable statements: register reductions in RTS ---*/
             e = &SgAssignOp(*new SgVarRefExp(s_tmp_var), *new SgValueExp(ln+1));
             stmt = new SgCExpStmt(*e);
@@ -2209,6 +2447,7 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
         stmt = makeSymbolDeclarationWithInit(s, new SgValueExp(0));
         st_hedr->insertStmtAfter(*stmt, *st_hedr);
     }
+
     // create indxs
     for (int i = 0; i < acrossV; ++i)
     {
@@ -2556,7 +2795,7 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
         e = &SgAssignOp(*new SgVarRefExp(s_blocks), *f);
         stmt = new SgCExpStmt(*e);
         st_end->insertStmtBefore(*stmt, *st_hedr);
-        stmt->addComment("//Start method");
+        stmt->addComment("// Start method");
 
         e = &SgAssignOp(*new SgVarRefExp(acrossBase[0]), *new SgArrayRefExp(*lowI, *new SgValueExp(loopAcrossSymb[0].len)));
         stmt = new SgCExpStmt(*e);
@@ -2722,7 +2961,6 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
                 st_end->insertStmtBefore(*stmt, *st_hedr);
             }
         }
-
         mywarn("start: in adding args section");
 
         /* args for kernel */
@@ -2781,6 +3019,35 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
                     sdev = sdev->next();
                 }
             }
+            
+            e_all_private_size = sizeOfPrivateArraysInBytes();
+            if (options.isOn(C_CUDA) && e_all_private_size)
+            {
+                for (el=private_list, lnp=0; el; el=el->rhs())
+                {
+                     s = el->lhs()->symbol();
+                     if (IS_ARRAY(s))
+                     {                
+                         sarg = new SgSymbol(VARIABLE_NAME, PointerNameForPrivateArray(s), *C_PointerType(C_VoidType()), *st_hedr);
+                         ae = new SgCastExp(*C_PointerType( C_Type(s->type()->baseType())), *new SgVarRefExp(sarg));
+                         funcCallKernel->addArg(*ae);
+                         if (!lnp)
+                             private_first = sarg;
+                         lnp++;        
+                         if (!TestArrayShape(s))
+                         {   
+                             SgExpression **eatr = (SgExpression **) el->lhs()->attributeValue(0, DIM_SIZES);  
+                             SgExpression *ela;
+                             for (ela = *eatr; ela; ela = ela->rhs())
+                                 funcCallKernel->addArg(SgDerefOp(*new SgVarRefExp(ela->lhs()->lhs()->symbol())));  
+                             eatr = (SgExpression **) el->lhs()->attributeValue(0, L_BOUNDS);    
+                             for (ela = *eatr; ela; ela = ela->rhs())
+                                 funcCallKernel->addArg(SgDerefOp(*new SgVarRefExp(ela->lhs()->lhs()->symbol()))); 
+                         }
+                     }
+                 }
+            }         
+
             funcCallKernel->addArg(*new SgVarRefExp(type_of_run));
             for (int i = 0; i < acrossV + loopV; ++i)
                 funcCallKernel->addArg(*new SgArrayRefExp(*bIdxs, *new SgValueExp(i)));
@@ -2816,7 +3083,15 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
             SgForStmt *simple;
             simple = new SgForStmt(&SgAssignOp(*new SgVarRefExp(tmpV), *new SgValueExp(0)), &(*new SgVarRefExp(tmpV1) < *new SgArrayRefExp(*highI, *new SgValueExp(loopAcrossSymb[0].len))), expr, stmt);
             st_end->insertStmtBefore(*simple);
+            stmt = simple;
         }
+        stmt->addComment("// GPU execution");
+        if (options.isOn(C_CUDA) && e_all_private_size)
+        {
+            e_totalThreads = &(*new SgRecordRefExp(*s_blocks, "x") * *new SgRecordRefExp(*s_blocks, "y") * *new SgRecordRefExp(*s_blocks, "z") * *new SgRecordRefExp(*s_threads, "x") * *new SgRecordRefExp(*s_threads, "y") * *new SgRecordRefExp(*s_threads, "z"));
+            GetMemoryForPrivateArrays(private_first, s_loop_ref, lnp, stmt, st_hedr, e_totalThreads);
+        }
+
     }
     else if (acrossV == 2) // ACROSS with two dependence: generate method
     {
@@ -2972,7 +3247,8 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
 
             mywarn("  end: out red section");
         }
-
+        createPrivatePointers(private_first, lnp, st_hedr, e_all_private_size);
+        GetMemoryForPrivateArrays (private_first, s_loop_ref, lnp, st_end, st_hedr, new SgVarRefExp(q));
         mywarn("strat: init bases");
         // init bases
         for (int i = 0; i < acrossV; ++i)
@@ -3014,7 +3290,7 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
                                       reduction_ptr, reduction_loc_ptr, reduction_symb, reduction_loc_symb, red_blocks,
                                       has_red_array, diag, loopV, num_elems, acrossV, acrossBase, loopBase, idxI,
                                       loopAcrossSymb, loopSymb, s, uses_first, sdev, scalar_first, uses_num, dvm_array_headers,
-                                      addressingParams, outTypeOfTransformation, type_of_run, bIdxs);
+                                      addressingParams, outTypeOfTransformation, type_of_run, bIdxs, private_first, lnp);
 
         stmt = createKernelCallsInCudaHandler(funcCallKernel, s_loop_ref, idxTypeInKernel, s_blocks);
         while_st->insertStmtAfter(*stmt);
@@ -3093,7 +3369,7 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
                                       reduction_ptr, reduction_loc_ptr, reduction_symb, reduction_loc_symb, red_blocks,
                                       has_red_array, q, loopV, num_elems, acrossV, acrossBase, loopBase, idxI,
                                       loopAcrossSymb, loopSymb, s, uses_first, sdev, scalar_first, uses_num, dvm_array_headers,
-                                      addressingParams, outTypeOfTransformation, type_of_run, bIdxs);
+                                      addressingParams, outTypeOfTransformation, type_of_run, bIdxs, private_first, lnp);
 
         while_st1->insertStmtAfter(*createKernelCallsInCudaHandler(funcCallKernel, s_loop_ref, idxTypeInKernel, s_blocks));
         while_st2->insertStmtAfter(*createKernelCallsInCudaHandler(funcCallKernel, s_loop_ref, idxTypeInKernel, s_blocks));
@@ -3105,7 +3381,7 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
                                       reduction_ptr, reduction_loc_ptr, reduction_symb, reduction_loc_symb, red_blocks,
                                       has_red_array, elem, loopV, num_elems, acrossV, acrossBase, loopBase, idxI,
                                       loopAcrossSymb, loopSymb, s, uses_first, sdev, scalar_first, uses_num, dvm_array_headers,
-                                      addressingParams, outTypeOfTransformation, type_of_run, bIdxs);
+                                      addressingParams, outTypeOfTransformation, type_of_run, bIdxs, private_first, lnp);
 
         while_st3->insertStmtAfter(*createKernelCallsInCudaHandler(funcCallKernel, s_loop_ref, idxTypeInKernel, s_blocks));
         while_st4->insertStmtAfter(*createKernelCallsInCudaHandler(funcCallKernel, s_loop_ref, idxTypeInKernel, s_blocks));
@@ -3190,6 +3466,30 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
                     sdev = sdev->next();
                 }
             }
+            createPrivatePointers(private_first, lnp, st_hedr, e_all_private_size);
+            if (options.isOn(C_CUDA) && private_first) // there are big private arrays
+            {
+                SgSymbol *sp;
+                for (sp = private_first, el = private_list, ln = 0;  ln < lnp; sp = sp->next(), el = el->rhs(), ln++) 
+                {            
+                    while (!IS_ARRAY(el->lhs()->symbol()))
+                        el = el->rhs();
+                    s = el->lhs()->symbol(); 
+                    ae = new SgCastExp(*C_PointerType( C_Type(s->type()->baseType())), *new SgVarRefExp(sp));
+                    funcCallKernel->addArg(*ae);
+                    if (!TestArrayShape(s))
+                    {   
+                        SgExpression **eatr = (SgExpression **) el->lhs()->attributeValue(0, DIM_SIZES);  
+                        SgExpression *ela;
+                        for (ela = *eatr; ela; ela = ela->rhs())
+                            funcCallKernel->addArg(SgDerefOp(*new SgVarRefExp(ela->lhs()->lhs()->symbol())));  
+                        eatr = (SgExpression **) el->lhs()->attributeValue(0, L_BOUNDS);    
+                        for (ela = *eatr; ela; ela = ela->rhs())
+                            funcCallKernel->addArg(SgDerefOp(*new SgVarRefExp(ela->lhs()->lhs()->symbol()))); 
+                    }
+                }
+            }         
+
             funcCall = new SgFunctionCallExp(*createNewFunctionSymbol("MIN"));
             funcCall->addArg(*new SgVarRefExp(M1));
             funcCall->addArg(*new SgVarRefExp(M2));
@@ -3392,6 +3692,18 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
 
             mywarn("  end: out red section");
         }
+        
+        if (options.isOn(C_CUDA) && private_first) 
+        {
+            SgFunctionCallExp *f1 = new SgFunctionCallExp(*createNewFunctionSymbol("MAX"));
+            SgFunctionCallExp *f2 = new SgFunctionCallExp(*createNewFunctionSymbol("MAX"));
+            f1->addArg(*new SgVarRefExp(M1));
+            f1->addArg(*new SgVarRefExp(M2));
+            f2->addArg(*f1);
+            f2->addArg(*new SgVarRefExp(M3));
+            e_totalThreads = &(*new SgVarRefExp(Emin) * *f2);
+            GetMemoryForPrivateArrays (private_first, s_loop_ref, lnp, st_end, st_hedr, e_totalThreads);
+        }
 
         int flag_comment = 0;
         for (int i = 3; i < acrossV; ++i)
@@ -3755,6 +4067,13 @@ vector<ArgsForKernel> Create_C_Adapter_Function_Across_variants(SgSymbol *sadapt
             st_end->insertStmtBefore(*stmt, *st_hedr);
         }
     }
+    // to dispose private arrays
+    if (options.isOn(C_CUDA) && e_all_private_size)
+        for (s = private_first, ln = 0;  ln < lnp; s = s->next(), ln++)  // private arrays
+        {
+            stmt = new SgCExpStmt(*DisposePrivateArray(s_loop_ref, s));
+	    st_end->insertStmtBefore(*stmt, *st_hedr);  
+        }
 
     // create args for kernel and return it
     vector<ArgsForKernel> argsKernel(countKernels);
@@ -3903,7 +4222,7 @@ void MakeDeclarationsForKernel_On_C_Across(SgType *indexType)
     DeclareDoVars(indexType);
 
     // declare private(local in kernel) variables
-    DeclarePrivateVars();
+    DeclarePrivateVars(indexType);
 
     // declare  variables, used in loop and passed by reference:
     // <type> &<name> = *p_<name>;
@@ -3920,7 +4239,7 @@ void MakeDeclarationsForKernelAcross(SgType *indexType)
     DeclareDoVars();
 
     // declare private(local in kernel) variables
-    DeclarePrivateVars();
+    DeclarePrivateVars(indexType);
 
     // declare dummy arguments:
 
@@ -3976,6 +4295,9 @@ SgExpression *CreateKernelDummyListAcross(ArgsForKernel *argsKer, SgType *idxTyp
     if (uses_list)
         arg_list = AddListToList(arg_list, CreateUsesDummyList()); //[+ <uses> ]
 
+    if (private_list)
+        arg_list = AddListToList(arg_list, CreatePrivateDummyList()); //[+ dummys for private arrays ]
+
     if (argsKer->symb.size() >= 3)
         for (int it = 0; it < argsKer->sizeVars.size(); ++it)
             arg_list = AddListToList(arg_list, new SgExprListExp(*new SgVarRefExp(argsKer->sizeVars[it])));
@@ -5513,6 +5835,7 @@ SgStatement *CreateLoopKernelAcross(SgSymbol *skernel, ArgsForKernel* argsKer, i
                 flag_func_call = 0; // maxloc
             else if (num == 10)
                 flag_func_call = 0; // minloc
+
             if (flag_func_call == 1)
             {
                 SgFunctionCallExp *funcCall = new SgFunctionCallExp(*createNewFunctionSymbol(str_operation));
@@ -5607,9 +5930,9 @@ SgStatement *CreateLoopKernelAcross(SgSymbol *skernel, ArgsForKernel* argsKer, i
                         locGrid->setType(*new SgArrayType(*tmp_list->loc_grid->type()));
 
                         if (options.isOn(C_CUDA))
-                            st = AssignStatement(*new SgArrayRefExp(*locGrid, *new SgValueExp(i), *e1), *new SgArrayRefExp(*loc_var_ref->symbol(), *new SgValueExp(i)));
+                            st = AssignStatement(*new SgArrayRefExp(*locGrid, *new SgValueExp(loc_el_num) * *e1 + *new SgValueExp(i)), *new SgArrayRefExp(*loc_var_ref->symbol(), *new SgValueExp(i)));
                         else
-                            st = AssignStatement(*new SgArrayRefExp(*locGrid, *new SgValueExp(i + 1), *e1), *new SgArrayRefExp(*loc_var_ref->symbol(), *new SgValueExp(i + 1)));
+                            st = AssignStatement(*new SgArrayRefExp(*locGrid, *new SgValueExp(i + 1), *e1), *new SgArrayRefExp(*loc_var_ref->symbol(), *new SgValueExp(i + 1)));//TODO it like in C_Cuda
                         ifSt->insertStmtAfter(*st);
                     }
                 }
@@ -5802,18 +6125,17 @@ SgSymbol *RedBlockSymbolInKernelAcross(SgSymbol *s, SgType *type)
 
 void DeclarationOfReductionBlockInKernelAcross(SgExpression *ered, reduction_operation_list *rsl)
 {
-    SgStatement *ass, *newst, *current, *if_st, *while_st, *typedecl, *st, *do_st;
-    SgExpression *le, *re, *eatr, *cond, *ev;
+    SgStatement *newst, *current, *if_st, *while_st, *typedecl, *st, *do_st;
+    SgExpression *eatr, *cond, *ev;
     SgSymbol *red_var, *red_var_k, *s_block, *loc_var, *sf;
     SgType *rtype;
-    int i, ind;
 
     //init block
-    ass = newst = current = if_st = while_st = typedecl = st = do_st = NULL;
-    le = re = eatr = cond = ev = NULL;
+    newst = current = if_st = while_st = typedecl = st = do_st = NULL;
+    eatr = cond = ev = NULL;
     red_var = red_var_k = s_block = loc_var = sf = NULL;
     rtype = NULL;
-    i = ind = loc_el_num = 0;
+    loc_el_num = 0;
     //end of init block
 
     // analys of reduction operation
@@ -5838,13 +6160,13 @@ void DeclarationOfReductionBlockInKernelAcross(SgExpression *ered, reduction_ope
 
     if (rsl->locvar)
     {
-        newst = Declaration_Statement(rsl->locvar); //declare location variable
+        newst = Declaration_Statement(LocRedVariableSymbolInKernel(rsl)); //declare location variable
         kernel_st->insertStmtAfter(*newst, *kernel_st);
     }
 
     if (rsl->redvar_size > 0)
     {
-        newst = Declaration_Statement(rsl->redvar); //declare reduction variable
+        newst = Declaration_Statement(RedVariableSymbolInKernel(rsl->redvar,NULL,NULL)); //declare reduction variable
         kernel_st->insertStmtAfter(*newst, *kernel_st);
     }
     else if (rsl->redvar_size < 0)
@@ -5853,26 +6175,22 @@ void DeclarationOfReductionBlockInKernelAcross(SgExpression *ered, reduction_ope
         newst = Declaration_Statement(red_var_k); //declare reduction variable
         kernel_st->insertStmtAfter(*newst, *kernel_st);
     }
-    rtype = (rsl->redvar_size >= 0) ? TypeOfRedBlockSymbol(ered) : red_var_k->type();
 
-    s_block = RedBlockSymbolInKernelAcross(red_var, rtype);
-
-    newst = Declaration_Statement(s_block);
-
-    if (options.isOn(C_CUDA))
-        newst->addDeclSpec(BIT_CUDA_SHARED | BIT_EXTERN);
-    else
+    //XXX: shared memory doesnt use in ACROSS by C_Cuda
+    if (!options.isOn(C_CUDA))
     {
+        rtype = (rsl->redvar_size >= 0) ? TypeOfRedBlockSymbol(ered) : red_var_k->type();
+        s_block = RedBlockSymbolInKernelAcross(red_var, rtype);
+        newst = Declaration_Statement(s_block);
         eatr = new SgExprListExp(*new SgExpression(ACC_SHARED_OP));
-        newst->setExpression(2, *eatr);
-    }
+        newst->setExpression(2, *eatr);    
+        kernel_st->insertStmtAfter(*newst, *kernel_st);
 
-    kernel_st->insertStmtAfter(*newst, *kernel_st);
-
-    if (isSgExprListExp(ered->rhs())) //MAXLOC,MINLOC
-    {
-        typedecl = MakeStructDecl(rtype->symbol());
-        kernel_st->insertStmtAfter(*typedecl, *kernel_st);
+        if (isSgExprListExp(ered->rhs())) //MAXLOC,MINLOC
+        {
+            typedecl = MakeStructDecl(rtype->symbol());
+            kernel_st->insertStmtAfter(*typedecl, *kernel_st);
+        } 
     }
 }
 

dvm/fdvm/trunk/fdvm/acc_across_analyzer.cpp

@@ -5,10 +5,12 @@
 using namespace std;
 
 // special storages to avoid recomputing
-map<string, SgExpression*> lhs;
-map<string, SgExpression*> rhs;
-map<SgExpression*, string> unparsedLhs;
-map<SgExpression*, string> unparsedRhs;
+static map<string, SgExpression*> lhs;
+static map<string, SgExpression*> rhs;
+static map<SgExpression*, string> unparsedLhs;
+static map<SgExpression*, string> unparsedRhs;
+
+extern reduction_operation_list* red_struct_list;
 
 template<typename InIt1, typename InIt2, typename OutIt>
 static inline OutIt difference(InIt1 first1, InIt1 last1, InIt2 first2, InIt2 last2, OutIt dest)
@@ -1220,7 +1222,14 @@ void Loop::analyzeInderectAccess()
 Loop::Loop(SgStatement* loop_body, bool enable_opt, bool irreg_access) : 
     irregular_acc_opt(irreg_access), enable_opt(enable_opt), loop_body(loop_body), 
     dimension(0), acrossType(0), acrossDims(NULL), do_irreg_opt(false)
-{    
+{
+    reduction_operation_list* rsl;
+    for (rsl = red_struct_list; rsl; rsl = rsl->next)
+    {
+        if (rsl->locvar)  //MAXLOC,MINLOC
+            redArrays.insert(rsl->locvar);
+    }
+
     lhs.clear();
     rhs.clear();
     unparsedLhs.clear();
@@ -1420,7 +1429,7 @@ void Loop::analyzeAssignments(SgExpression* ex, const int blockIndex)
     else
     {
         SgSymbol* symbol = ex->symbol();
-        if (isSgArrayType(symbol->type()) != NULL)
+        if (isSgArrayType(symbol->type()) != NULL && redArrays.find(symbol) == redArrays.end())
         {
             SgExpression* subscripts = ((SgArrayRefExp*)(ex))->subscripts();
             if (!subscripts)
@@ -1949,7 +1958,14 @@ void Loop::buildCFG()
 }
 
 Loop::Loop(SgStatement* stmt) : do_irreg_opt(false)
-{    
+{
+    reduction_operation_list* rsl;
+    for (rsl = red_struct_list; rsl; rsl = rsl->next)
+    {
+        if (rsl->locvar)  //MAXLOC,MINLOC
+            redArrays.insert(rsl->locvar);
+    }
+
     lhs.clear(); rhs.clear(); unparsedLhs.clear(); unparsedRhs.clear();
     buildCFG();
 }

dvm/fdvm/trunk/fdvm/funcall.cpp

@@ -4941,4 +4941,49 @@ SgExpression *RtcSetLang(SgSymbol *s_loop_ref, const int lang)
     else
         fe->addArg(*new SgKeywordValExp("UNKNOWN_CUDA")); 
     return(fe);
-}
+}
+
+SgExpression *GetDeviceProp(SgSymbol *s_loop_ref, SgExpression *ep)
+{// generating function call: 
+ //                         DvmType loop_cuda_get_device_prop(DvmType *InDvmhLoop, DvmType prop);
+                        
+    SgFunctionCallExp *fe = new SgFunctionCallExp(*fdvm[GET_DEVICE_PROP]);
+
+    fe->addArg(*new SgVarRefExp(s_loop_ref));
+    fe->addArg(*ep);
+    return(fe);
+}
+
+SgExpression *GetMaxBlocks(SgSymbol *s_loop_ref, SgSymbol *s_max_blocks, SgSymbol *s_needed_bytes)
+{// generating function call: 
+ //                         DvmType loop_cuda_get_max_blocks(DvmType *InDvmhLoop, DvmType maxBlocks, DvmType neededBytesForBlock) 
+
+    SgFunctionCallExp *fe = new SgFunctionCallExp(*fdvm[GET_MAX_BLOCKS]);
+
+    fe->addArg(*new SgVarRefExp(s_loop_ref));
+    fe->addArg(*new SgVarRefExp(s_max_blocks));
+    fe->addArg(*new SgVarRefExp(s_needed_bytes));
+    return(fe);
+}
+
+SgExpression *GetPrivateArray(SgSymbol *s_loop_ref, SgExpression *e_bytes)
+{// generating function call: 
+ //                         DvmType *loop_cuda_get_private_array(DvmType *InDvmhLoop, UDvmType neededBytes)
+
+    SgFunctionCallExp *fe = new SgFunctionCallExp(*fdvm[GET_PRIVATE_ARR]);
+
+    fe->addArg(*new SgVarRefExp(s_loop_ref));
+    fe->addArg(*e_bytes);
+    return(fe);
+}
+
+SgExpression *DisposePrivateArray(SgSymbol *s_loop_ref, SgSymbol *s_array)
+{// generating function call: 
+ //                         void loop_cuda_dispose_private_array(DvmType *InDvmhLoop, void *array)
+
+    SgFunctionCallExp *fe = new SgFunctionCallExp(*fdvm[DISPOSE_PRIVATE_AR]);
+
+    fe->addArg(*new SgVarRefExp(s_loop_ref));
+    fe->addArg(*new SgVarRefExp(s_array));
+    return(fe);
+}

dvm/fdvm/trunk/include/acc_across_analyzer.h

@@ -104,6 +104,7 @@ private:
     SgStatement* loop_body;
     int dimension;
     std::map<SgSymbol*, Array*> arrays;
+    std::set<SgSymbol*> redArrays;
     int* acrossDims;
     int acrossType;
     std::vector<SgSymbol*> symbols;

dvm/fdvm/trunk/include/dvm.h

@@ -261,6 +261,10 @@ const int END_OF_USE_LIST   = 1050; /*ACC*/
 const int END_OF_USE_LIST   = 1050; /*ACC*/
 const int ROUTINE_ATTR      = 1051; /*ACC*/
 const int DATA_REGION_SYMB  = 1052; /*ACC*/
+const int REMOTE_ACCESS_BUF = 1053; /*ACC*/
+const int L_BOUNDS          = 1054; /*ACC*/
+const int DIM_SIZES         = 1055; /*ACC*/
+const int PRIVATE_ARRAY     = 1056; /*ACC*/
 const int PRIVATE_POINTER   = 1057; /*ACC*/
 
 const int MAX_LOOP_LEVEL = 20; // 7 - maximal number of loops in parallel loop nest 
@@ -1275,6 +1279,7 @@ SgSymbol *isSameRedVar(char *name);
 SgSymbol *isSameRedVar(char *name);
 SgSymbol *isSameArray(char *name);
 SgSymbol *isSameIndexVar(char *name);
+SgType * C_LongLongType();
 SgType * C_UnsignedLongLongType();
 SgType * C_DvmType();
 SgType * C_CudaIndexType();
@@ -1372,6 +1377,7 @@ int TestOneGroupStatement(SgStatement *stmt);
 int TestOneGroupStatement(SgStatement *stmt);
 void DeclareUsedVars();
 void DeclareInternalPrivateVars();
+void DeclarePrivateVars();
 void DeclarePrivateVars(SgType *idxTypeInKernel);
 void DeclareArrayBases();
 void DeclareArrayCoeffsInKernel(SgType*);
@@ -1434,6 +1440,19 @@ SgSymbol *HeaderSymbolForHandler(SgSymbol *ar);
 SgSymbol *HeaderSymbolForHandler(SgSymbol *ar);
 void TestRoutineAttribute(SgSymbol *s, SgStatement *routine_interface);
 int LookForRoutineDir(SgStatement *interfaceFunc);
+SgStatement *Interface(SgSymbol *s);
+SgExpression *sizeOfElementInBytes(SgSymbol *symb);
+SgExpression *sizeOfPrivateArraysInBytes();
+SgExpression *ProductOfDimSizeArgs(SgExpression *esizes);
+//void doPrivateArrayList(SgExpression *private_arrays, SgStatement *st_hedr);
+void addPrivateArrayList(SgFunctionCallExp *fcall, SgExpression *private_arrays, SgStatement *st_hedr);
+int TestArrayShape(SgSymbol *ar);
+SgExpression *DimSizeListOfPrivateArrays();
+SgExpression *BoundListOfPrivateArrays();
+SgExpression * DummyListForPrivateArrays(SgStatement *st_hedr);
+SgExpression *CreatePrivateDummyList();
+char *PointerNameForPrivateArray(SgSymbol *symb);
+void GetMemoryForPrivateArrays(SgSymbol *private_first, SgSymbol *s_loop_ref, int nump, SgStatement *st_end, SgStatement *st_hedr, SgExpression *e_totalThreads);
 SgSymbol *LocRedVariableSymbolInKernel(reduction_operation_list *rsl);
 
 /*   acc_analyzer.cpp   */
@@ -1900,6 +1919,10 @@ SgStatement *Consistent_H (int il, SgExpression *hedr, SgExpression *axis_list);
 SgStatement *Consistent_H (int il, SgExpression *hedr, SgExpression *axis_list);
 SgStatement *LoopRemoteAccess_H (int il, SgExpression *hedr, SgSymbol *ar, SgExpression *axis_list);
 SgStatement *RemoteAccess_H2 (SgExpression *buf_hedr, SgSymbol *ar, SgExpression *ar_hedr, SgExpression *axis_list);
+SgStatement *GetRemoteBuf (SgSymbol *loop_s, int n, SgSymbol *s_buf_head);
+SgExpression *GetDeviceProp(SgSymbol *s_loop_ref, SgExpression *ep);
+SgExpression *GetMaxBlocks(SgSymbol *s_loop_ref, SgSymbol *s_max_blocks, SgSymbol *s_needed_bytes);
+SgExpression *GetPrivateArray(SgSymbol *s_loop_ref, SgExpression *e_bytes);
 SgExpression *DisposePrivateArray(SgSymbol *s_loop_ref, SgSymbol *s_array);
 
 /*  io.cpp      */
@@ -2089,7 +2112,6 @@ char *Check_Correct_Name(const char *name);
 
 /* acc_f2c.cpp */
 void Translate_Fortran_To_C(SgStatement *stat, SgStatement *last, std::vector <std::stack <SgStatement*> > &, int);
-SgStatement* Translate_Fortran_To_C(SgStatement* Stmt, bool isSapforConv = false);
 SgStatement* Translate_Fortran_To_C(SgStatement* Stmt, bool isSapforConv = false);
 SgSymbol* createNewFunctionSymbol(const char *name);
 void swapDimentionsInprivateList(void);
@@ -2103,6 +2125,9 @@ void RenamingNewProcedureVariables(SgSymbol *proc_name);
 void RenamingNewProcedureVariables(SgSymbol *proc_name);
 SgSymbol *hasSameNameAsSource(SgSymbol *symb);
 void RenamingCudaFunctionVariables(SgStatement *first, SgSymbol *k_symb, int replace_flag);
+void replaceVariableSymbSameNameInStatements(SgStatement *first, SgStatement *last, SgSymbol *symb, SgSymbol *s_new, int replace_flag);
+void RenamingCalledProcedureSymbols(SgStatement *header, SgStatement *copy_header);
+void RenamingCalledProcedureSymbolsInKernel(SgSymbol *first_symb);
 
 /* acc_across.cpp */
 ArgsForKernel *Create_C_Adapter_Function_Across(SgSymbol *sadapter);
@@ -2238,7 +2263,7 @@ void ConvertLoopWithLabelToEnddoLoop (SgStatement *stat); /*OMP*/
 // options on FDVM converter
 enum OPTIONS {
     AUTO_TFM = 0, ONE_THREAD, SPEED_TEST_L0, SPEED_TEST_L1, GPU_O0, GPU_O1, RTC, C_CUDA, OPT_EXP_COMP,
-    O_HOST, NO_CUDA, NO_BL_INFO, LOOP_ANALYSIS, PRIVATE_ANALYSIS, IO_RTS, READ_ALL, NO_REMOTE, NO_PURE_FUNC, 
+    O_HOST, NO_CUDA, NO_BL_INFO, LOOP_ANALYSIS, PRIVATE_ANALYSIS, IO_RTS, READ_ALL, NO_REMOTE, NO_PURE_FUNC, 
     GPU_IRR_ACC, O_PL, O_PL2, BIG_P, NUM_OPT};
 // ONE_THREAD - compile one thread CUDA-kernels only for across (TODO for all CUDA-kernels)
 // SPEED_TEST_L0, SPEED_TEST_L1 - debug options for speed testof CUDA-kernels for across

dvm/fdvm/trunk/include/dvm_tag.h

@@ -62,7 +62,8 @@
 #define DVM_CP_WAIT_DIR              638
 #define DVM_EXIT_INTERVAL_DIR	     639
 #define DVM_TEMPLATE_CREATE_DIR	     640 
-#define DVM_TEMPLATE_DELETE_DIR	     641       		   
+#define DVM_TEMPLATE_DELETE_DIR	     641 
+#define PRIVATE_AR_DECL              642         		   
 #define BLOCK_OP           705
 #define NEW_SPEC_OP        706
 #define REDUCTION_OP       707

dvm/fdvm/trunk/include/libdvm.h

@@ -334,3 +334,7 @@ name_dvm[GUESS_INDEX_TYPE] = "loop_guess_index_type_";
 name_dvm[GUESS_INDEX_TYPE_2]="dvmh_loop_guess_index_type_C";
 name_dvm[RTC_SET_LANG]     = "loop_cuda_rtc_set_lang";
 name_dvm[GET_REMOTE_BUF_C] = "dvmh_loop_get_remote_buf_C";
+name_dvm[GET_DEVICE_PROP]  = "loop_cuda_get_device_prop";
+name_dvm[GET_MAX_BLOCKS]   = "loop_cuda_get_max_blocks";
+name_dvm[GET_PRIVATE_ARR]  = "loop_cuda_get_private_array";
+name_dvm[DISPOSE_PRIVATE_AR]="loop_cuda_dispose_private_array";

dvm/fdvm/trunk/include/libnum.h

@@ -332,5 +332,9 @@ enum {
       GUESS_INDEX_TYPE_2,
       RTC_SET_LANG,
       GET_REMOTE_BUF_C,
+      GET_DEVICE_PROP,
+      GET_MAX_BLOCKS,
+      GET_PRIVATE_ARR,
+      DISPOSE_PRIVATE_AR,
       MAX_LIBFUN_NUM
 };  

dvm/fdvm/trunk/parser/tag

@@ -236,6 +236,7 @@
 #define DVM_EXIT_INTERVAL_DIR	 639    /* DVM-F */
 #define DVM_TEMPLATE_CREATE_DIR	 640    /* DVM-F */ 
 #define DVM_TEMPLATE_DELETE_DIR	 641    /* DVM-F */
+#define PRIVATE_AR_DECL          642    /* DVM-F */
     
 /***************** variant tags for low level nodes ********************/
 

dvm/fdvm/trunk/parser/tag.h

@@ -238,7 +238,8 @@ script using "tag".  Run make tag.h to regenerate this file */
   tag [ DVM_EXIT_INTERVAL_DIR ] = "DVM_EXIT_INTERVAL_DIR";
   tag [ DVM_TEMPLATE_CREATE_DIR ] = "DVM_TEMPLATE_CREATE_DIR";
   tag [ DVM_TEMPLATE_DELETE_DIR ] = "DVM_TEMPLATE_DELETE_DIR";
-    
+  tag [ PRIVATE_AR_DECL ] = "PRIVATE_AR_DECL";
+  
 /***************** variant tags for low level nodes ********************/
 
   tag [ INT_VAL ] = "INT_VAL";

sapfor/experts/Sapfor_2017/CMakeLists.txt

@@ -193,6 +193,11 @@ set(TR_CONV _src/Transformations/convert_to_c.cpp
         _src/Transformations/convert_to_c.h)
 set(TR_IMPLICIT_NONE _src/Transformations/set_implicit_none.cpp
         _src/Transformations/set_implicit_none.h)
+set(TR_REPLACE_ARRAYS_IN_IO _src/Transformations/replace_dist_arrays_in_io.cpp
+        _src/Transformations/replace_dist_arrays_in_io.h)
+set(FIND_PRIVATE_ARRAYS _src/PrivateArrays/private_arrays_search.cpp
+        _src/PrivateArrays/private_arrays_search.h)
+
 
 set(TRANSFORMS 
         ${TR_DEAD_CODE}
@@ -211,7 +216,8 @@ set(TRANSFORMS
         ${TR_PRIV_DEL}
         ${TR_CONV}
         ${TR_PRIV_DEL}
-        ${TR_IMPLICIT_NONE})
+        ${TR_IMPLICIT_NONE}
+        ${TR_REPLACE_ARRAYS_IN_IO})
         
 set(CFG _src/CFGraph/IR.cpp
         _src/CFGraph/IR.h
@@ -254,6 +260,7 @@ set(DIRA  _src/DirectiveProcessing/directive_analyzer.cpp
         _src/DirectiveProcessing/spf_directive_preproc.cpp)
         
 set(DISTR _src/Distribution/Array.cpp
+        _src/Distribution/ArrayAnalysis.cpp
         _src/Distribution/Array.h
         _src/Distribution/Arrays.h
         _src/Distribution/CreateDistributionDirs.cpp
@@ -316,7 +323,7 @@ set(LOOP_ANALYZER  _src/LoopAnalyzer/allocations_prepoc.cpp
 
 set(RENAME_SYMBOLS  _src/RenameSymbols/rename_symbols.cpp
                     _src/RenameSymbols/rename_symbols.h)
-        
+
 
 set(MAIN _src/Sapfor.cpp
         _src/Sapfor.h
@@ -419,7 +426,8 @@ set(SOURCE_EXE
     ${ZLIB}
     ${GR_LAYOUT}
     ${PR_PARAM}
-    ${PROJ_MAN})
+    ${PROJ_MAN}
+    ${FIND_PRIVATE_ARRAYS})
 
 add_executable(Sapfor_F ${SOURCE_EXE})
 source_group (CFGraph FILES ${CFG})
@@ -444,6 +452,7 @@ source_group (Transformations\\RenameSymbols FILES ${RENAME_SYMBOLS})
 source_group (Transformations\\GlobalVariables FILES ${TR_GV})
 source_group (Transformations\\ConvertToC FILES ${TR_CONV})
 source_group (Transformations\\SetImplicitNone FILES ${TR_IMPLICIT_NONE})
+source_group (Transformations\\ReplaceArraysInIO FILES ${TR_REPLACE_ARRAYS_IN_IO})
 
 
 source_group (CreateIntervals FILES ${CREATE_INTER_T})
@@ -471,6 +480,8 @@ source_group (Predictor FILES ${PREDICTOR})
 source_group (Parser FILES ${PARSER})
 source_group (PPPA\\PPPA FILES ${PPPA})
 source_group (PPPA\\ZLib FILES ${ZLIB})
+
+source_group (PrivateArrays FILES ${FIND_PRIVATE_ARRAYS})
    
 if (MSVC_IDE)   
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP /Zc:__cplusplus")

sapfor/experts/Sapfor_2017/_src/CFGraph/IR.h

@@ -162,6 +162,35 @@ namespace SAPFOR
 
         SgExpression* getExpression() const { return ex; }
 
+        int getLine() const { return st ? st->lineNumber() : -1; }
+
+        bool isAccess() const 
+        {
+            std::set<CFG_OP> accessOps = { CFG_OP::LOAD , CFG_OP::STORE, CFG_OP::REC_REF_LOAD, CFG_OP::REC_REF_STORE };
+            return accessOps.find(operation) != accessOps.end();
+        }
+
+        bool isArith() const
+        {
+            std::set<CFG_OP> arithOps = { CFG_OP::ADD, CFG_OP::MULT, CFG_OP::DIV, CFG_OP::SUBT, CFG_OP::UN_ADD, CFG_OP::UN_MINUS, CFG_OP::POW, CFG_OP::CONCAT };
+                                         //CFG_OP::CAST, CFG_OP::GE, CFG_OP::LE, CFG_OP::GT, CFG_OP::LT, CFG_OP::EQ, CFG_OP::NEQV, CFG_OP::EQV, CFG_OP::OR, CFG_OP::AND, CFG_OP::NOT };
+
+            return (arithOps.find(operation) != arithOps.end()) || isIntirinsicCall();
+        }
+
+        bool isIntirinsicCall() const
+        {
+            if (operation == CFG_OP::F_CALL)
+            {
+                if (ex)
+                {
+                    if (isIntrinsicFunctionName(ex->symbol()->identifier()))
+                        return true;
+                }
+            }
+            return false;
+        }
+
         static int getNextInstrNum() { return lastNumInstr; }
         static void shiftNextInstrNum(int byNum) { lastNumInstr += byNum; }
 
@@ -296,6 +325,8 @@ namespace SAPFOR
         void setHeader() { header = true; }
         bool isHeader() const { return header; }
 
+        int getLine() const { return current->getLine(); }
+
         ~IR_Block() { delete current; }
     };
 

sapfor/experts/Sapfor_2017/_src/DirectiveProcessing/directive_parser.cpp

@@ -122,15 +122,20 @@ extern map<string, vector<Messages>> SPF_messages;
 
 //for simple reduction
 template<typename fillType>
-void fillReductionsFromComment(Statement *stIn, map<string, set<fillType>> &reduction, bool moduleNameAdd)
+void fillReductionsFromComment(Statement *stIn, map<string, set<fillType>> &reduction, bool moduleNameAdd, int type)
 {    
     bool error = false;
     if (stIn)
     {
         SgStatement *st = stIn->GetOriginal();
-        if (st->variant() == SPF_ANALYSIS_DIR)
+        if (st->variant() == type)
         {
-            SgExpression *exprList = st->expr(0);
+            SgExpression* exprList = NULL;
+            if (type == SPF_ANALYSIS_DIR)
+                exprList = st->expr(0);
+            else if (type == DVM_PARALLEL_ON_DIR)
+                exprList = st->expr(1);
+
             while (exprList)
             {
                 if (exprList->lhs()->variant() == REDUCTION_OP)
@@ -188,19 +193,24 @@ void fillReductionsFromComment(Statement *stIn, map<string, set<fillType>> &redu
         printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
 }
 
-template void fillReductionsFromComment(Statement *st, map<string, set<string>> &reduction, bool);
-template void fillReductionsFromComment(Statement *st, map<string, set<Symbol*>> &reduction, bool);
+template void fillReductionsFromComment(Statement *st, map<string, set<string>> &reduction, bool, int);
+template void fillReductionsFromComment(Statement *st, map<string, set<Symbol*>> &reduction, bool, int);
 
 //for min/max loc reduction
 template<typename fillType>
-void fillReductionsFromComment(Statement *stIn, map<string, set<tuple<fillType, fillType, int>>> &reduction, bool moduleNameAdd)
+void fillReductionsFromComment(Statement *stIn, map<string, set<tuple<fillType, fillType, int>>> &reduction, bool moduleNameAdd, int type)
 {
     if (stIn)
     {
         SgStatement *st = stIn->GetOriginal();
-        if (st->variant() == SPF_ANALYSIS_DIR)
+        if (st->variant() == type)
         {
-            SgExpression *exprList = st->expr(0);
+            SgExpression* exprList = NULL;
+            if (type == SPF_ANALYSIS_DIR)
+                exprList = st->expr(0);
+            else if (type == DVM_PARALLEL_ON_DIR)
+                exprList = st->expr(1);
+
             while (exprList)
             {
                 if (exprList->lhs()->variant() == REDUCTION_OP)
@@ -241,8 +251,8 @@ void fillReductionsFromComment(Statement *stIn, map<string, set<tuple<fillType,
     }
 }
 
-template void fillReductionsFromComment(Statement *st, map<string, set<tuple<string, string, int>>> &reduction, bool);
-template void fillReductionsFromComment(Statement *st, map<string, set<tuple<Symbol*, Symbol*, int>>> &reduction, bool);
+template void fillReductionsFromComment(Statement *st, map<string, set<tuple<string, string, int>>> &reduction, bool, int);
+template void fillReductionsFromComment(Statement *st, map<string, set<tuple<Symbol*, Symbol*, int>>> &reduction, bool, int);
 
 void fillParameterFromComment(Statement *stIn, vector<pair<Expression*, Expression*>> &assigns)
 {
@@ -598,6 +608,28 @@ void fillInfoFromDirectives(const LoopGraph *loopInfo, ParallelDirective *direct
     }
 }
 
+void fillInfoFromDirective(Statement* parallel_on, DvmDirective& directive)
+{
+    Statement* sData = parallel_on;
+
+    fillReductionsFromComment(sData, directive.reduction, true, DVM_PARALLEL_ON_DIR);
+    fillReductionsFromComment(sData, directive.reductionLoc, true, DVM_PARALLEL_ON_DIR);
+    
+    fillShadowAcrossFromParallel(SHADOW_RENEW_OP, sData, directive.shadowRenew, directive.corners);
+    fillShadowAcrossFromParallel(ACROSS_OP, sData, directive.across, directive.corners);
+
+    //TODO:
+    /*map<pair<Symbol*, string>, Expression*> remotes;
+    fillRemoteFromComment(sData, remotes, false, DVM_PARALLEL_ON_DIR);
+    for (auto& elem : remotes)
+    {
+        SgSymbol* symb = OriginalSymbol(elem.first.first->GetOriginal());
+        auto uniqKey = getFromUniqTable(symb);
+
+        directive.remoteAccess[make_pair(make_pair(elem.first.first->GetOriginal()->identifier(), getShortName(uniqKey)), elem.first.second)] = new Expression(elem.second->GetOriginal());
+    }*/
+}
+
 int getCoverPropertyFromComment(Statement* stIn)
 {
     if (stIn)

sapfor/experts/Sapfor_2017/_src/DirectiveProcessing/directive_parser.h

@@ -8,16 +8,26 @@
 #include "../GraphLoop/graph_loops.h"
 #include "../Distribution/DvmhDirective.h"
 
+struct DvmDirective
+{
+    std::set<Symbol*> corners;
+    std::vector<std::pair<std::pair<Symbol*, std::string>, std::vector<std::pair<int, int>>>> shadowRenew, across;
+    std::map<std::pair<std::pair<std::string, std::string>, std::string>, Expression*> remoteAccess;
+
+    std::map<std::string, std::set<Symbol*>> reduction;
+    std::map<std::string, std::set<std::tuple<Symbol*, Symbol*, int>>> reductionLoc;
+};
+
 bool isSPF_NoInline(Statement *stPrev);
 
 template<typename fillType>
 void fillPrivatesFromComment(Statement *st, std::set<fillType> &privates, int type = -1);
 
 template<typename fillType>
-void fillReductionsFromComment(Statement *st, std::map<std::string, std::set<fillType>> &reduction, bool moduleNameAdd = false);
+void fillReductionsFromComment(Statement *st, std::map<std::string, std::set<fillType>> &reduction, bool moduleNameAdd = false, int type = SPF_ANALYSIS_DIR);
 
 template<typename fillType>
-void fillReductionsFromComment(Statement *st, std::map<std::string, std::set<std::tuple<fillType, fillType, int>>> &reduction, bool moduleNameAdd = false);
+void fillReductionsFromComment(Statement *st, std::map<std::string, std::set<std::tuple<fillType, fillType, int>>> &reduction, bool moduleNameAdd = false, int type = SPF_ANALYSIS_DIR);
 
 void fillParameterFromComment(Statement *st, std::vector<std::pair<Expression*, Expression*>> &assigns);
 
@@ -32,6 +42,7 @@ void fillRemoteFromComment(Statement *st, std::map<std::pair<fillType, std::stri
 
 void fillAcrossInfoFromDirectives(const LoopGraph *loopInfo, std::vector<std::pair<std::pair<std::string, std::string>, std::vector<std::pair<int, int>>>> &acrossInfo);
 void fillInfoFromDirectives(const LoopGraph *loopInfo, ParallelDirective *directive);
+void fillInfoFromDirective(Statement* parallel_on, DvmDirective& directive);
 
 void fillFissionPrivatesExpansionFromComment(Statement *stIn, std::vector<std::string> &vars);
 

sapfor/experts/Sapfor_2017/_src/DirectiveProcessing/remote_access.cpp

@@ -23,6 +23,7 @@
 #include "../Utils/SgUtils.h"
 #include "../Distribution/Arrays.h"
 #include "../GraphCall/graph_calls.h"
+#include "../GraphCall/graph_calls_func.h"
 #include "../GraphLoop/graph_loops_func.h"
 #include "remote_access.h"
 
@@ -863,8 +864,16 @@ void addRemoteLink(const LoopGraph* loop, const map<string, FuncInfo*>& funcMap,
     }
 }
 
-ArrayRefExp* createRemoteLink(const DIST::Array* forArray)
+ArrayRefExp* createRemoteLink(const LoopGraph* currLoop, const DIST::Array* forArray)
 {
+    SgFile* file = current_file;
+
+    const set<string> allFiles = getAllFilesInProject();
+    SgStatement* realStat = (SgStatement*)currLoop->getRealStat(file->filename());
+    const map<string, set<SgSymbol*>> byUseInFunc = moduleRefsByUseInFunction(realStat);
+    SgStatement* parentFunc = getFuncStat(realStat);
+    const pair<int, int> lineRange = make_pair(parentFunc->lineNumber(), parentFunc->lastNodeOfStmt()->lineNumber());
+
     SgExpression* ex = new SgExpression(EXPR_LIST);
     SgExpression* p = ex;
     for (int z = 0; z < forArray->GetDimSize(); ++z)
@@ -880,21 +889,17 @@ ArrayRefExp* createRemoteLink(const DIST::Array* forArray)
 
     auto decls = forArray->GetDeclInfoWithSymb();
     const string fName = current_file->filename();
-    for (auto& decl : decls)
+    /*for (auto& decl : decls)
     {
         if (decl.first.first == fName)
         {
             newRem = new SgArrayRefExp(*decl.second->GetOriginal(), *ex);
             break;
         }
-    }
+    }*/
 
     if (!newRem)
-    {
-        auto arrayT = new SgType(T_ARRAY);
-        auto tmpS = findSymbolOrCreate(current_file, forArray->GetShortName(), arrayT);
-        newRem = new SgArrayRefExp(*tmpS, *ex);
-    }
+        newRem = new SgArrayRefExp(*getFromModule(byUseInFunc, forArray->GetDeclSymbol(fName, lineRange, allFiles)->GetOriginal()), *ex);
     return new ArrayRefExp(newRem);
 }
 

sapfor/experts/Sapfor_2017/_src/DirectiveProcessing/remote_access.h

@@ -29,7 +29,7 @@ void addRemoteLink(const LoopGraph* loop, const std::map<std::string, FuncInfo*>
                    const std::set<std::string>& remotesInParallel, std::set<ArrayRefExp*>& addedRemotes, const std::vector<std::string>& mapToLoop,
                    std::vector<Messages>& messages, const int line, bool bindToLoopDistribution = true);
 
-ArrayRefExp* createRemoteLink(const DIST::Array* forArray);
+ArrayRefExp* createRemoteLink(const LoopGraph* currLoop, const DIST::Array* forArray);
 
 std::vector<RemoteRequest> 
      checkArrayRefInLoopForRemoteStatus(bool ifUnknownArrayAssignFound,

sapfor/experts/Sapfor_2017/_src/DirectiveProcessing/remote_access_base.cpp

@@ -86,7 +86,7 @@ static void createRemoteInParallel(const pair<LoopGraph*, const ParallelDirectiv
                     mapToLoop.push_back("");
 
                 //TODO: find all array refs 
-                addRemoteLink(under_dvm_dir.first, funcMap, createRemoteLink(usedArr), uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, -1, false);
+                addRemoteLink(under_dvm_dir.first, funcMap, createRemoteLink(under_dvm_dir.first, usedArr), uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, -1, false);
                 __spf_print(DEB, "CRIP: %d, AFTER MAIN CHECK for arrray '%s'\n", __LINE__, usedArr->GetShortName().c_str());
             }
         }

sapfor/experts/Sapfor_2017/_src/DirectiveProcessing/shadow.cpp

@@ -309,7 +309,7 @@ static void replaceShadowByRemote(SgExpression *specInDir, SgStatement *stat,
 
 void devourShadowByRemote(void *file_, const map<string, FuncInfo*>& funcMap, const vector<LoopGraph*>& loops, 
                           const map<DIST::Array*, set<DIST::Array*>> &arrayLinksByFuncCalls)
-{    
+{
     SgFile* file = static_cast<SgFile*>(file_);
     map<int, LoopGraph*> loopByLine;
     createMapLoopGraph(loops, loopByLine);

sapfor/experts/Sapfor_2017/_src/Distribution/Array.cpp

@@ -122,7 +122,7 @@ namespace Distribution
         return retVal;
     }
 
-    void TemplateLink::AddRule(const int dimNum, int dimTempl, const pair<int, int> &rule, Array *templateArray_)
+    void TemplateLink::AddRule(const int dimNum, int dimTempl, const pair<int, int>& rule, Array* templateArray_)
     {
         linkWithTemplate[dimNum] = dimTempl;
         alignRuleWithTemplate[dimNum] = rule;
@@ -132,7 +132,7 @@ namespace Distribution
     const vector<pair<int, int>>& Array::GetSizes()
     {
         if (templateDimsOrder.size() == 0)
-            return sizes; 
+            return sizes;
         else
         {
             if (orderedSizes.size() == 0)
@@ -141,7 +141,7 @@ namespace Distribution
         }
     }
 
-    const vector<pair<pair<Expression*, pair<int, int>>, pair<Expression*, pair<int, int>>>>& 
+    const vector<pair<pair<Expression*, pair<int, int>>, pair<Expression*, pair<int, int>>>>&
         Array::GetSizesExpr()
     {
         if (templateDimsOrder.size() == 0)
@@ -154,7 +154,7 @@ namespace Distribution
         }
     }
 
-    void printArrayInfo(const string &file, const map<tuple<int, string, string>, pair<Array*, ArrayAccessInfo*>> &declaredArrays)
+    void printArrayInfo(const string& file, const map<tuple<int, string, string>, pair<Array*, ArrayAccessInfo*>>& declaredArrays)
     {
         FILE* out = fopen(file.c_str(), "w");
         if (out == NULL)
@@ -180,7 +180,7 @@ namespace Distribution
                 locN = "LOCAL (save) of ";
             else if (loc.first == l_STRUCT)
                 locN = "STRUCT ";
-            else 
+            else
                 locN = "UNKN ";
             locN += "'" + loc.second + "'";
 
@@ -189,7 +189,7 @@ namespace Distribution
             {
                 fprintf(out, " FOR FILE '%s':\n", convertFileName(byFile.first.c_str()).c_str());
                 for (auto& byLine : byFile.second)
-                    for (auto &onLine : byLine.second)
+                    for (auto& onLine : byLine.second)
                         fprintf(out, "  %s\n", onLine.PrintInfo().c_str());
             }
             fprintf(out, "=============== \n\n");
@@ -198,8 +198,8 @@ namespace Distribution
         fclose(out);
     }
 
-    void fixTypeOfArrayInfoWithCallGraph(map<tuple<int, string, string>, pair<Array*, ArrayAccessInfo*>> &declaredArrays,
-                                         const map<string, FuncInfo*> &allFuncs)
+    void fixTypeOfArrayInfoWithCallGraph(map<tuple<int, string, string>, pair<Array*, ArrayAccessInfo*>>& declaredArrays,
+        const map<string, FuncInfo*>& allFuncs)
     {
         for (auto& elem : declaredArrays)
         {

sapfor/experts/Sapfor_2017/_src/Distribution/ArrayAnalysis.cpp

@@ -0,0 +1,766 @@
+#include "Array.h"
+#include "../Utils/errors.h"
+#include "../Utils/utils.h"
+#include "../GraphCall/graph_calls.h"
+#include "../Utils/SgUtils.h"
+#include "../DirectiveProcessing/directive_parser.h"
+#include "../DirectiveProcessing/directive_omp_parser.h"
+#include "../LoopAnalyzer/loop_analyzer.h"
+
+using namespace std;
+
+extern int ignoreIO;
+extern map<DIST::Array*, std::tuple<int, string, string>> tableOfUniqNamesByArray;
+static set<tuple<int, string, string>> checkedArraysForWrongLocation;
+
+static bool findOmpThreadPrivDecl(SgStatement* st, map<SgStatement*, set<string>>& ompThreadPrivate, SgSymbol* toFind)
+{
+
+    auto it = ompThreadPrivate.find(st);
+    if (it == ompThreadPrivate.end())
+    {
+        it = ompThreadPrivate.insert(it, make_pair(st, set<string>()));
+
+        SgStatement* lastN = st->lastNodeOfStmt();
+        set<string> dummy;
+        do
+        {
+            st = st->lexNext();
+
+            auto res = parseOmpInStatement(st, dummy);
+            for (auto& dir : res)
+                for (auto& var : dir.threadPrivVars)
+                    it->second.insert(var);
+        } while (st != lastN && !isSgExecutableStatement(st) && st->variant() != CONTAINS_STMT);
+    }
+
+    if (it->second.find(toFind->identifier()) != it->second.end())
+        return true;
+    else
+        return false;
+}
+
+static bool hasAssingOpInDecl(SgSymbol* symb)
+{
+    vector<SgStatement*> allDecls;
+    SgStatement* decl = declaratedInStmt(symb, &allDecls);
+
+    for (auto& elem : allDecls)
+    {
+        if (elem->variant() == VAR_DECL_90)
+        {
+            SgExpression* list = elem->expr(0);
+            while (list)
+            {
+                if (list->lhs()->variant() == ASSGN_OP)
+                    if (list->lhs()->lhs()->symbol() && OriginalSymbol(list->lhs()->lhs()->symbol()) == symb)
+                        return true;
+                list = list->rhs();
+            }
+        }
+    }
+
+    return false;
+}
+
+static string getNameWithScope(SgStatement* scope, const string& currFunctionName)
+{
+    if (scope && isSgProgHedrStmt(scope) && scope->symbol()->identifier() != currFunctionName)
+        return scope->symbol()->identifier();
+    else
+        return currFunctionName;
+}
+
+struct findInfo
+{
+    findInfo(const string fName, SgExpression* ex, int parN, bool isWrite) :
+        fName(fName), ex(ex), parN(parN), isWrite(isWrite)
+    { }
+
+    SgExpression* ex;
+    string fName;
+    int parN;
+    bool isWrite;
+};
+
+static void findArrayRefs (SgExpression* ex, SgStatement* st, string fName, int parN, bool isWrite,
+                           const map<string, vector<SgExpression*>>& commonBlocks,
+                           map<tuple<int, string, string>, pair<DIST::Array*, DIST::ArrayAccessInfo*>>& declaredArrays,
+                           map<SgStatement*, set<tuple<int, string, string>>>& declaratedArraysSt,
+                           const set<string>& privates, const set<string>& deprecatedByIO,
+                           bool isExecutable, const string& currFunctionName,
+                           const vector<string>& inRegion,
+                           const set<string>& funcParNames,
+                           map<SgStatement*, set<string>>& ompThreadPrivate,
+                           const map<string, int>& distrStateFromGUI,
+                           const bool saveAllLocals,
+                           map<string, vector<Messages>>& currMessages,
+                           int& errorCount)
+{
+    const string globalFile = current_file->filename();
+    const set<string> filesInProj = getAllFilesInProject();
+
+    if (ex == NULL)
+        return;
+    stack<findInfo> queue;
+    queue.push(findInfo(fName, ex, parN, isWrite));
+
+    while (!queue.empty())
+    {
+        const findInfo& curQ = queue.top();
+        ex = curQ.ex;
+        fName = curQ.fName;
+        parN = curQ.parN;
+        isWrite = curQ.isWrite;
+
+        queue.pop();
+
+        if (isArrayRef(ex))
+        {
+            SgSymbol* symb = OriginalSymbol(ex->symbol());
+            const bool inDataStat = (symb->attributes() & DATA_BIT) != 0 || ((ex->symbol())->attributes() & DATA_BIT);
+            checkNull(symb->type(), convertFileName(__FILE__).c_str(), __LINE__);
+
+            const int typeSize = getSizeOfType(symb->type()->baseType());
+            if (typeSize == 0)
+            {
+                //__spf_print(1, "Wrong type size for array %s\n", symb->identifier());
+                //printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+            }
+
+            SgStatement* decl = declaratedInStmt(symb);
+            if (decl->variant() == DVM_VAR_DECL || decl->variant() == HPF_TEMPLATE_STAT)
+            {
+                const string tmp(decl->unparse());
+                if (tmp.find("!DVM$ TEMPLATE") != string::npos)
+                {
+                    auto sTemp = symb->identifier();
+                    tuple<int, string, string> uniqKey;
+                    bool found = false;
+                    for (auto& elem : declaredArrays)
+                    {
+                        if (elem.second.first->GetShortName() == sTemp)
+                        {
+                            uniqKey = elem.first;
+                            found = true;
+                        }
+                    }
+
+                    if (found)
+                    {
+                        auto itDecl = declaratedArraysSt.find(decl);
+                        if (itDecl == declaratedArraysSt.end())
+                            itDecl = declaratedArraysSt.insert(itDecl, make_pair(decl, set<tuple<int, string, string>>()));
+                        itDecl->second.insert(uniqKey);
+
+                        return;
+                    }
+                }
+            }
+
+            auto uniqKey = getUniqName(commonBlocks, decl, symb);
+
+            SgStatement* scope = symb->scope();
+            pair<DIST::arrayLocType, string> arrayLocation;
+
+            string typePrefix = "";
+            while (scope && scope->variant() == STRUCT_DECL)
+            {
+                if (typePrefix == "")
+                    typePrefix = scope->symbol()->identifier();
+                else
+                    typePrefix += scope->symbol()->identifier() + string("::");
+                scope = scope->controlParent();
+            }
+
+            if ((ex->symbol() && symb != ex->symbol()) || (scope && scope->variant() == MODULE_STMT))
+            {
+                if (scope)
+                {
+                    string modName = scope->symbol()->identifier();
+                    arrayLocation = make_pair(DIST::l_MODULE, (typePrefix == "") ? modName : modName + "::" + typePrefix);
+                }
+                else //TODO: find module name with another way
+                    arrayLocation = make_pair(DIST::l_MODULE, "UNREC_MODULE_NAME");
+            }
+            else if (get<1>(uniqKey).find("common_") != string::npos)
+                arrayLocation = make_pair(DIST::l_COMMON, get<1>(uniqKey).substr(strlen("common_")));
+            else if (funcParNames.find(symb->identifier()) != funcParNames.end())
+                arrayLocation = make_pair(DIST::l_PARAMETER, getNameWithScope(scope, currFunctionName));
+            else
+            {
+                if (saveAllLocals || ((symb->attributes() & SAVE_BIT) != 0) || hasAssingOpInDecl(symb))
+                    arrayLocation = make_pair(DIST::l_LOCAL_SAVE, getNameWithScope(scope, currFunctionName));
+                else
+                    arrayLocation = make_pair(DIST::l_LOCAL, getNameWithScope(scope, currFunctionName));
+            }
+
+            auto itNew = declaredArrays.find(uniqKey);
+            if (itNew == declaredArrays.end())
+            {
+                DIST::Array* arrayToAdd =
+                    new DIST::Array(getShortName(uniqKey), symb->identifier(), ((SgArrayType*)(symb->type()))->dimension(),
+                        getUniqArrayId(), decl->fileName(), decl->lineNumber(), arrayLocation, new Symbol(symb),
+                        findOmpThreadPrivDecl(scope, ompThreadPrivate, symb), false, false,
+                        inRegion, typeSize, sharedMemoryParallelization ? DIST::NO_DISTR : DIST::DISTR);
+
+                itNew = declaredArrays.insert(itNew, make_pair(uniqKey, make_pair(arrayToAdd, new DIST::ArrayAccessInfo())));
+
+                vector<pair<int, int>> sizes;
+                map<DIST::Array*, set<DIST::Array*>> arrayLinksByFuncCallsNotReady;
+                map<string, vector<FuncInfo*>> allFuncInfoNoReady;
+                auto sizesExpr = getArraySizes(sizes, symb, decl, arrayLinksByFuncCallsNotReady, allFuncInfoNoReady);
+                arrayToAdd->SetSizes(sizes);
+                arrayToAdd->SetSizesExpr(sizesExpr);
+                tableOfUniqNamesByArray[arrayToAdd] = uniqKey;
+            }
+            else // check the same location from include!
+            {
+                auto prevLocation = itNew->second.first->GetLocation();
+                if (prevLocation != arrayLocation)
+                {
+                    if (checkedArraysForWrongLocation.find(uniqKey) == checkedArraysForWrongLocation.end())
+                    {
+                        checkedArraysForWrongLocation.insert(uniqKey);
+                        __spf_print(1, "can not change declaration area of array '%s' on line %d\n", symb->identifier(), st->lineNumber());
+
+                        wstring messageE, messageR;
+                        __spf_printToLongBuf(messageE, L"Array '%s' has declaration area conflict, it might be worth applying the Include inlining pass", to_wstring(symb->identifier()).c_str());
+                        __spf_printToLongBuf(messageR, R184, to_wstring(symb->identifier()).c_str());
+
+                        currMessages[st->fileName()].push_back(Messages(ERROR, st->lineNumber(), messageR, messageE, 1061));
+                    }
+                    errorCount++;
+                }
+            }
+
+            if ((symb->attributes() & EQUIVALENCE_BIT) != 0)
+                itNew->second.first->SetEquvalence(true);
+
+            for (auto& reg : inRegion)
+                itNew->second.first->SetRegionPlace(reg);
+
+            const auto oldVal = itNew->second.first->GetDistributeFlagVal();
+            bool isarrayInModule = (itNew->second.first->GetLocation().first == DIST::l_MODULE);
+
+            if (oldVal == DIST::DISTR || oldVal == DIST::NO_DISTR)
+            {
+                if (itNew->second.first->IsOmpThreadPrivate())
+                    itNew->second.first->SetDistributeFlag(DIST::SPF_PRIV);
+                else if (privates.find(symb->identifier()) != privates.end() || isarrayInModule)
+                {
+                    //check in module
+                    if (itNew->second.first->GetLocation().first == DIST::l_MODULE)
+                    {
+                        for (auto& data : getAttributes<SgStatement*, SgStatement*>(decl, set<int>{ SPF_ANALYSIS_DIR }))
+                        {
+                            set<string> privatesS;
+                            fillPrivatesFromComment(new Statement(data), privatesS);
+                            if (privatesS.find(symb->identifier()) != privatesS.end())
+                            {
+                                itNew->second.first->SetDistributeFlag(DIST::SPF_PRIV);
+                                break;
+                            }
+                        }
+
+                        auto prev = decl->lexPrev();
+                        checkNull(prev, convertFileName(__FILE__).c_str(), __LINE__);
+
+                        set<string> privatesS;
+                        fillPrivatesFromComment(new Statement(prev), privatesS);
+                        if (privatesS.find(symb->identifier()) != privatesS.end())
+                            itNew->second.first->SetDistributeFlag(DIST::SPF_PRIV);
+                    }
+                    else
+                        itNew->second.first->SetDistributeFlag(DIST::SPF_PRIV);
+                }
+                else if (deprecatedByIO.find(symb->identifier()) != deprecatedByIO.end())
+                    itNew->second.first->SetDistributeFlag(DIST::IO_PRIV);
+                else if (isSgConstantSymb(symb) || inDataStat)
+                    itNew->second.first->SetDistributeFlag(DIST::SPF_PRIV);
+                else
+                {
+                    auto it = distrStateFromGUI.find(itNew->second.first->GetIndepUniqName());
+                    if (it != distrStateFromGUI.end())
+                    {
+                        if (it->second != oldVal)
+                        {
+                            itNew->second.first->SetDistributeFlag((DIST::distFlag)it->second);
+                            __spf_print(1, "change flag for array from cache '%s': %d -> %d\n", it->first.c_str(), oldVal, it->second);
+                        }
+                    }
+                    else
+                        itNew->second.first->SetDistributeFlag(DIST::DISTR);
+                }
+            }
+
+            if (typeSize == 0) // unknown
+                itNew->second.first->SetDistributeFlag(DIST::SPF_PRIV);
+
+
+            if (!isExecutable)
+                itNew->second.first->AddDeclInfo(make_pair(st->fileName(), st->lineNumber()), filesInProj, globalFile, new Symbol(symb));
+
+            if (isExecutable)
+            {
+                if (st->variant() != ALLOCATE_STMT && st->variant() != DEALLOCATE_STMT)
+                {
+                    itNew->second.second->AddAccessInfo(st->fileName(), make_pair(st->lineNumber(), isWrite ? 1 : 0), fName, parN);
+                    itNew->second.first->AddUsagePlace(st->fileName(), st->lineNumber());
+                }
+            }
+
+            auto itDecl = declaratedArraysSt.find(decl);
+            if (itDecl == declaratedArraysSt.end())
+                itDecl = declaratedArraysSt.insert(itDecl, make_pair(decl, set<tuple<int, string, string>>()));
+            itDecl->second.insert(uniqKey);
+
+            if (decl->variant() == DVM_VAR_DECL || decl->variant() == HPF_TEMPLATE_STAT)
+            {
+                const string tmp(decl->unparse());
+                if (tmp.find("!DVM$ TEMPLATE") != string::npos)
+                {
+                    itNew->second.first->SetTemplateFlag(true);
+                    //TODO: analyze align mapping
+                    for (int z = 0; z < itNew->second.first->GetDimSize(); ++z)
+                        itNew->second.first->SetMappedDim(z);
+                }
+            }
+        }
+
+        if (ex->variant() == FUNC_CALL)
+        {
+            SgFunctionCallExp* funcExp = (SgFunctionCallExp*)ex;
+            const string fName = funcExp->funName()->identifier();
+            bool intr = (isIntrinsicFunctionName(fName.c_str()) == 1);
+            for (int z = 0; z < funcExp->numberOfArgs(); ++z)
+            {
+                //assume all arguments of function as OUT, except for inctrinsics
+                bool isWriteN = intr ? false : true;
+                //need to correct W/R usage with GraphCall map later
+                findArrayRefs(funcExp->arg(z), st, fName, z, isWriteN, commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO, isExecutable, currFunctionName, inRegion, funcParNames, ompThreadPrivate, distrStateFromGUI, saveAllLocals, currMessages, errorCount);
+            }
+        }
+        else
+        {
+            bool isWriteN = false;
+            if (ex->lhs())
+                queue.push(findInfo("", ex->lhs(), -1, isWriteN));
+            if (ex->rhs())
+                queue.push(findInfo("", ex->rhs(), -1, isWriteN));
+            //findArrayRefs(ex->lhs(), st, "", -1, isWriteN, commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO, isExecutable, currFunctionName, inRegion, funcParNames, ompThreadPrivate, distrStateFromGUI, saveAllLocals, currMessages, errorCount);
+            //findArrayRefs(ex->rhs(), st, "", -1, isWriteN, commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO, isExecutable, currFunctionName, inRegion, funcParNames, ompThreadPrivate, distrStateFromGUI, saveAllLocals, currMessages, errorCount);
+        }
+    }
+}
+
+static void findArrayRefInIO(SgExpression* ex, set<string>& deprecatedByIO, SgStatement* st, map<string, vector<Messages>>& currMessages)
+{
+    if (ex)
+    {
+        if (ex->variant() == ARRAY_REF)
+        {
+            auto symb = ex->symbol();
+            if (symb->type())
+            {
+                if (symb->type()->variant() == T_ARRAY)
+                {
+                    auto found = deprecatedByIO.find(OriginalSymbol(symb)->identifier());
+                    if (found == deprecatedByIO.end())
+                    {
+                        deprecatedByIO.insert(found, OriginalSymbol(symb)->identifier());
+
+                        wstring messageE, messageR;
+                        __spf_printToLongBuf(messageE, L"Array '%s' can not be distributed because of DVM's I/O constraints", to_wstring(symb->identifier()).c_str());
+                        __spf_printToLongBuf(messageR, R68, to_wstring(symb->identifier()).c_str());
+
+                        currMessages[st->fileName()].push_back(Messages(WARR, st->lineNumber(), messageR, messageE, 1037));
+
+                        __spf_print(1, "Array '%s' at line %d can not be distributed because of DVM's I/O constraints\n", symb->identifier(), st->lineNumber());
+                    }
+                }
+            }
+        }
+
+        findArrayRefInIO(ex->lhs(), deprecatedByIO, st, currMessages);
+        findArrayRefInIO(ex->rhs(), deprecatedByIO, st, currMessages);
+    }
+}
+
+static void findReshape(SgStatement* st, set<string>& privates, map<string, vector<Messages>>& currMessages)
+{
+    if (st->variant() == ASSIGN_STAT)
+    {
+        SgExpression* exL = st->expr(0);
+        SgExpression* exR = st->expr(1);
+
+        if (exR->variant() == FUNC_CALL && exL->variant() == ARRAY_REF)
+        {
+            if (exR->symbol()->identifier() == string("reshape"))
+            {
+                if (privates.find(exL->symbol()->identifier()) == privates.end())
+                {
+                    privates.insert(exL->symbol()->identifier());
+
+                    wstring messageE, messageR;
+                    __spf_printToLongBuf(messageE, L"Array '%s' can not be distributed because of RESHAPE", to_wstring(exL->symbol()->identifier()).c_str());
+                    __spf_printToLongBuf(messageR, R90, to_wstring(exL->symbol()->identifier()).c_str());
+                    currMessages[st->fileName()].push_back(Messages(NOTE, st->lineNumber(), messageR, messageE, 1047));
+                }
+            }
+        }
+    }
+}
+
+static void findConstructorRef(SgStatement* st, set<string>& privates, map<string, vector<Messages>>& currMessages)
+{
+    if (st->variant() == ASSIGN_STAT)
+    {
+        SgExpression* exL = st->expr(0);
+        SgExpression* exR = st->expr(1);
+
+        if (exR->variant() == CONSTRUCTOR_REF && exL->variant() == ARRAY_REF)
+        {
+            if (privates.find(exL->symbol()->identifier()) == privates.end())
+            {
+                privates.insert(exL->symbol()->identifier());
+
+                wstring messageE, messageR;
+                __spf_printToLongBuf(messageE, L"Array '%s' can not be distributed because of initializer list", to_wstring(exL->symbol()->identifier()).c_str());
+                __spf_printToLongBuf(messageR, R164, to_wstring(exL->symbol()->identifier()).c_str());
+                currMessages[st->fileName()].push_back(Messages(NOTE, st->lineNumber(), messageR, messageE, 1047));
+            }
+        }
+    }
+}
+
+static void addPrivates(SgStatement* st, set<string>& privates, map<string, set<Symbol*>>& reductions,
+                        map<string, set<tuple<Symbol*, Symbol*, int>>>& reductionsLoc)
+{
+    //after SPF preprocessing 
+    for (auto& data : getAttributes<SgStatement*, SgStatement*>(st, set<int>{ SPF_ANALYSIS_DIR }))
+    {
+        set<Symbol*> privatesS;
+        fillPrivatesFromComment(new Statement(data), privatesS);
+        fillReductionsFromComment(new Statement(data), reductions);
+        fillReductionsFromComment(new Statement(data), reductionsLoc);
+
+        for (auto& elem : privatesS)
+            privates.insert(elem->GetOriginal()->identifier());
+    }
+
+    //before SPF preprocessing 
+    if (st->variant() == SPF_ANALYSIS_DIR)
+    {
+        set<Symbol*> privatesS;
+        fillPrivatesFromComment(new Statement(st), privatesS);
+        fillReductionsFromComment(new Statement(st), reductions);
+        fillReductionsFromComment(new Statement(st), reductionsLoc);
+
+        for (auto& elem : privatesS)
+            privates.insert(elem->GetOriginal()->identifier());
+    }
+}
+
+int getAllDeclaredArrays(SgFile* file, map<tuple<int, string, string>, pair<DIST::Array*, DIST::ArrayAccessInfo*>>& declaredArrays,
+                         map<SgStatement*, set<tuple<int, string, string>>>& declaratedArraysSt, map<string, vector<Messages>>& currMessages,
+                         const vector<ParallelRegion*>& regions, const map<string, int>& distrStateFromGUI)
+{
+    int countErrors = 0;
+
+    vector<SgStatement*> modules;
+    findModulesInFile(file, modules);
+
+    map<string, set<string>> privatesByModule;
+    for (auto& mod : modules)
+    {
+        const string modName = mod->symbol()->identifier();
+        privatesByModule[modName] = set<string>();
+        auto it = privatesByModule.find(modName);
+
+        for (SgStatement* iter = mod; iter != mod->lastNodeOfStmt(); iter = iter->lexNext())
+        {
+            if (iter->variant() == CONTAINS_STMT)
+                break;
+
+            //after SPF preprocessing 
+            for (auto& data : getAttributes<SgStatement*, SgStatement*>(iter, set<int>{ SPF_ANALYSIS_DIR }))
+                fillPrivatesFromComment(new Statement(data), it->second);
+
+            //before SPF preprocessing 
+            if (iter->variant() == SPF_ANALYSIS_DIR)
+                fillPrivatesFromComment(new Statement(iter), it->second);
+        }
+    }
+
+    map<SgStatement*, set<string>> ompThreadPrivate;
+
+    for (int i = 0; i < file->numberOfFunctions(); ++i)
+    {
+        bool saveAllLocals = false;
+
+        SgStatement* st = file->functions(i);
+        SgStatement* lastNode = st->lastNodeOfStmt();
+        map<string, vector<SgExpression*>> commonBlocks;
+        const string currFunctionName = st->symbol()->identifier();
+
+        getCommonBlocksRef(commonBlocks, st, lastNode);
+        set<string> privates;
+        set<string> deprecatedByIO;
+        map<string, set<Symbol*>> reductions;
+        map<string, set<tuple<Symbol*, Symbol*, int>>> reductionsLoc;
+        set<string> funcParNames;
+
+        if (st->variant() != PROG_HEDR)
+        {
+            SgProcHedrStmt* func = (SgProcHedrStmt*)st;
+            for (int z = 0; z < func->numberOfParameters(); ++z)
+                funcParNames.insert(func->parameter(z)->identifier());
+
+            if (func->nameWithContains() != func->name().identifier()) // added contains args
+            {
+                SgProcHedrStmt* cp = (SgProcHedrStmt*)func->controlParent();
+                checkNull(cp, convertFileName(__FILE__).c_str(), __LINE__);
+
+                for (int z = 0; z < cp->numberOfParameters(); ++z)
+                    funcParNames.insert(cp->parameter(z)->identifier());
+            }
+        }
+
+        for (SgStatement* iter = st; iter != lastNode; iter = iter->lexNext())
+        {
+            if (iter->variant() == CONTAINS_STMT)
+                break;
+
+            addPrivates(iter, privates, reductions, reductionsLoc);
+
+            if (iter->variant() == USE_STMT)
+                fillFromModule(iter->symbol(), privatesByModule, privates);
+
+            if (iter->variant() == SAVE_DECL)
+                if (!iter->expr(0) && !iter->expr(1) && !iter->expr(2))
+                    saveAllLocals = true;
+        }
+
+        for (SgStatement* iter = st; iter != lastNode; iter = iter->lexNext())
+        {
+            if (iter->variant() == CONTAINS_STMT)
+                break;
+
+            findReshape(iter, privates, currMessages);
+            findConstructorRef(iter, privates, currMessages);
+        }
+
+        SgStatement* tmpModFind = st;
+        while (tmpModFind->variant() != GLOBAL)
+        {
+            tmpModFind = tmpModFind->controlParent();
+            if (tmpModFind->variant() == MODULE_STMT)
+                fillFromModule(tmpModFind->symbol(), privatesByModule, privates);
+        }
+
+        SgStatement* currF = st;
+        SgStatement* contains = isSgProgHedrStmt(currF->controlParent());
+        if (contains)
+        {
+            for (SgStatement* loc = contains; loc; loc = loc->lexNext())
+            {
+                if (isSgExecutableStatement(loc))
+                    break;
+                if (loc->variant() == CONTAINS_STMT)
+                    break;
+
+                if (loc->variant() == USE_STMT)
+                    fillFromModule(loc->symbol(), privatesByModule, privates);
+            }
+        }
+
+        for (auto& elem : reductions)
+            for (auto& setElem : elem.second)
+                privates.insert(setElem->identifier());
+
+        for (auto& elem : reductionsLoc)
+        {
+            for (auto& setElem : elem.second)
+            {
+                privates.insert(get<0>(setElem)->identifier());
+                privates.insert(get<1>(setElem)->identifier());
+            }
+        }
+
+        //analyze IO operations
+        if (!ignoreIO)
+        {
+            for (SgStatement* iter = st; iter != lastNode; iter = iter->lexNext())
+            {
+                if (iter->variant() == CONTAINS_STMT)
+                    break;
+
+                SgInputOutputStmt* stIO = isSgInputOutputStmt(iter);
+                set<ParallelRegion*> currRegs = getAllRegionsByLine(regions, iter->fileName(), iter->lineNumber());
+                if (stIO && currRegs.size()) // deprecate to distribute arrays only in regions
+                {
+                    int countOfItems = 0;
+                    for (SgExpression* items = stIO->itemList(); items; items = items->rhs(), ++countOfItems);
+
+                    //TODO: need to add more checkers!
+                    if (countOfItems > 1)
+                    {
+                        for (SgExpression* items = stIO->itemList(); items; items = items->rhs())
+                            findArrayRefInIO(items->lhs(), deprecatedByIO, stIO, currMessages);
+                    }
+                    else if (countOfItems == 1)
+                    {
+                        auto list = stIO->specList();
+                        bool ok = true;
+                        //exclude FMT='format'
+                        while (list)
+                        {
+                            if (list->lhs() && list->lhs()->variant() == SPEC_PAIR)
+                            {
+                                auto ex = list->lhs();
+                                if (ex->lhs() && ex->rhs())
+                                {
+                                    if (ex->lhs()->variant() == KEYWORD_VAL)
+                                    {
+                                        SgKeywordValExp* key = (SgKeywordValExp*)(ex->lhs());
+                                        if (key->value() == string("fmt"))
+                                            if (ex->rhs()->variant() == STRING_VAL)
+                                                ok = false;
+                                    }
+                                }
+                            }
+
+                            if (!ok)
+                                break;
+                            list = list->rhs();
+                        }
+
+                        //check A(i,j) for example
+                        auto item = stIO->itemList()->lhs();
+                        if (item->rhs() || item->lhs())
+                            ok = false;
+
+                        if (!ok)
+                            findArrayRefInIO(item, deprecatedByIO, stIO, currMessages);
+                    }
+                }
+            }
+        }
+
+        while (st != lastNode)
+        {
+            if (st->variant() == CONTAINS_STMT)
+                break;
+
+            if (!isSPF_stat(st) && !isDVM_stat(st))
+            {
+                set<ParallelRegion*> currRegs = getAllRegionsByLine(regions, st->fileName(), st->lineNumber());
+                vector<string> regNames;
+                for (auto& reg : currRegs)
+                    regNames.push_back(reg->GetName());
+                if (regNames.size() == 0)
+                    regNames.push_back("default");
+
+                if (st->variant() == PROC_STAT)
+                {
+                    SgCallStmt* funcExp = (SgCallStmt*)st;
+                    const string fName = funcExp->symbol()->identifier();
+                    for (int z = 0; z < funcExp->numberOfArgs(); ++z)
+                    {
+                        findArrayRefs(funcExp->arg(z), st, fName, z, true,
+                            commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO,
+                            isSgExecutableStatement(st) ? true : false, currFunctionName,
+                            regNames, funcParNames, ompThreadPrivate, distrStateFromGUI, saveAllLocals,
+                            currMessages, countErrors);
+                    }
+                }
+                else
+                {
+                    for (int i = 0; i < 3; ++i)
+                        findArrayRefs(st->expr(i), st, "", -1, (st->variant() == ASSIGN_STAT && i == 0) ? true : false,
+                            commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO,
+                            isSgExecutableStatement(st) ? true : false, currFunctionName,
+                            regNames, funcParNames, ompThreadPrivate, distrStateFromGUI, saveAllLocals,
+                            currMessages, countErrors);
+                }
+            }
+            st = st->lexNext();
+        }
+    }
+
+    //preprocess only module declaration 
+    for (auto& mod : modules)
+    {
+        SgStatement* st = mod->lexNext();
+        SgStatement* lastNode = mod->lastNodeOfStmt();
+        map<string, vector<SgExpression*>> commonBlocks;
+        set<string> privates;
+        set<string> deprecatedByIO;
+        set<string> funcParNames;
+
+        fillFromModule(st->symbol(), privatesByModule, privates);
+
+        while (st != lastNode)
+        {
+            if (st->variant() == CONTAINS_STMT)
+                break;
+
+            if (!isSPF_stat(st) && !isDVM_stat(st))
+            {
+                //TODO: set clear regions for modules
+                set<ParallelRegion*> currRegs = getAllRegionsByLine(regions, st->fileName(), st->lineNumber());
+                vector<string> regNames;
+                for (auto& reg : currRegs)
+                    regNames.push_back(reg->GetName());
+                if (regNames.size() == 0)
+                    regNames.push_back("default");
+
+                for (int i = 0; i < 3; ++i)
+                    findArrayRefs(st->expr(i), st, "", -1, false, commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO,
+                        false, "NULL", regNames, funcParNames, ompThreadPrivate, distrStateFromGUI, false,
+                        currMessages, countErrors);
+            }
+            st = st->lexNext();
+        }
+    }
+
+    //preprocess only block data declaration 
+    for (SgStatement* st = file->firstStatement()->lexNext(); st; st = st->lastNodeOfStmt(), st = st->lexNext())
+    {
+        if (st->variant() == BLOCK_DATA)
+        {
+            SgStatement* last = st->lastNodeOfStmt();
+            SgStatement* curr = st;
+
+            map<string, vector<SgExpression*>> commonBlocks;
+            getCommonBlocksRef(commonBlocks, st, last);
+
+            set<string> privates;
+            set<string> deprecatedByIO;
+            set<string> funcParNames;
+
+            string blockName = "BLOCK DATA";
+            if (st->symbol())
+                blockName = st->symbol()->identifier();
+
+            while (curr && curr != last)
+            {
+                //TODO: set clear regions for block data
+                set<ParallelRegion*> currRegs = getAllRegionsByLine(regions, curr->fileName(), curr->lineNumber());
+                vector<string> regNames;
+                for (auto& reg : currRegs)
+                    regNames.push_back(reg->GetName());
+                if (regNames.size() == 0)
+                    regNames.push_back("default");
+
+                for (int i = 0; i < 3; ++i)
+                    findArrayRefs(curr->expr(i), curr, "", -1, false, commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO,
+                        false, blockName, regNames, funcParNames, ompThreadPrivate, distrStateFromGUI, false,
+                        currMessages, countErrors);
+                curr = curr->lexNext();
+            }
+        }
+    }
+    return countErrors;
+}

sapfor/experts/Sapfor_2017/_src/Distribution/DvmhDirective.cpp

@@ -197,15 +197,6 @@ static set<string> fillUsedSymbols(SgStatement *loop)
     return usedS;
 }
 
-static SgStatement* getRealStat(const char *file, const int line, const int altLine)
-{
-    SgStatement* local = SgStatement::getStatementByFileAndLine(file, line);
-    if (local == NULL)
-        local = SgStatement::getStatementByFileAndLine(file, altLine);
-    checkNull(local, convertFileName(__FILE__).c_str(), __LINE__);
-    return local;
-}
-
 static string correctSymbolModuleName(const string& origFull)
 {
     auto it = origFull.find("::");
@@ -512,7 +503,7 @@ ParallelDirective::genDirective(File* file, const vector<pair<DIST::Array*, cons
     vector<SgStatement*> moduleList;
     findModulesInFile(file, moduleList);
 
-    SgStatement* realStat = getRealStat(file->filename(), currLoop->lineNum, currLoop->altLineNum);
+    SgStatement* realStat = (SgStatement*)currLoop->getRealStat(file->filename());
     SgStatement* parentFunc = getFuncStat(realStat);
     const map<string, set<SgSymbol*>> byUseInFunc = moduleRefsByUseInFunction(realStat);
     const int nested = countPerfectLoopNest(loopG);

sapfor/experts/Sapfor_2017/_src/GraphCall/graph_calls.cpp

@@ -2513,7 +2513,7 @@ void fillInterfaceBlock(map<string, vector<FuncInfo*>>& allFuncInfo)
     for (auto& byFile : allFuncInfo)
     {
         for (auto& func : byFile.second)
-        {
+        {            
             for (auto& interface : func->interfaceBlocks)
             {
                 auto itF = mapOfFunc.find(interface.first);
@@ -2534,6 +2534,13 @@ void fillInterfaceBlock(map<string, vector<FuncInfo*>>& allFuncInfo)
                     }
                 }
             }
+
+            //filted interfaces
+            map<string, FuncInfo*> copy = func->interfaceBlocks;
+            func->interfaceBlocks.clear();
+            for (auto& interface : func->interfaceBlocks)
+                if (interface.second)
+                    func->interfaceBlocks[interface.first] = interface.second;
         }
     }
 

sapfor/experts/Sapfor_2017/_src/GraphLoop/graph_loops.cpp

@@ -908,6 +908,15 @@ void LoopGraph::analyzeParallelDirs()
         ch->analyzeParallelDirs();
 }
 
+void* LoopGraph::getRealStat(const char* file) const
+{
+    SgStatement* local = SgStatement::getStatementByFileAndLine(file, lineNum);
+    if (local == NULL)
+        local = SgStatement::getStatementByFileAndLine(file, altLineNum);
+    checkNull(local, convertFileName(__FILE__).c_str(), __LINE__);
+    return local;
+}
+
 extern int PASSES_DONE[EMPTY_PASS];
 static void printToBuffer(const LoopGraph *currLoop, const int childSize, char buf[512])
 {

sapfor/experts/Sapfor_2017/_src/GraphLoop/graph_loops.h

@@ -391,6 +391,8 @@ public:
 
     void analyzeParallelDirs();
 
+    void* getRealStat(const char* file) const;
+
 public:
     int lineNum;
     int altLineNum;

sapfor/experts/Sapfor_2017/_src/LoopAnalyzer/loop_analyzer.cpp

@@ -62,7 +62,6 @@ extern REGIME currRegime;
 extern std::vector<Messages>* currMessages;
 
 extern int sharedMemoryParallelization;
-extern int ignoreIO;
 extern int parallizeFreeLoops;
 
 using std::vector;
@@ -956,9 +955,9 @@ static SgExpression* replaceConstatantProcedurePars(SgExpression *dimList, SgSta
     return dimList;
 }
 
-static vector<pair<Expression*, Expression*>> getArraySizes(vector<pair<int, int>> &sizes, SgSymbol *symb, SgStatement *decl, 
-                                                            const map<DIST::Array*, set<DIST::Array*>> &arrayLinksByFuncCalls,
-                                                            const map<string, vector<FuncInfo*>> &allFuncInfo)
+vector<pair<Expression*, Expression*>> getArraySizes(vector<pair<int, int>> &sizes, SgSymbol *symb, SgStatement *decl, 
+                                                     const map<DIST::Array*, set<DIST::Array*>> &arrayLinksByFuncCalls,
+                                                     const map<string, vector<FuncInfo*>> &allFuncInfo)
 {
     SgArrayType *type = isSgArrayType(symb->type());
     vector<pair<Expression*, Expression*>> retVal;
@@ -2616,758 +2615,6 @@ int getSizeOfType(SgType *t)
     return len;
 }
 
-static bool findOmpThreadPrivDecl(SgStatement* st, map<SgStatement*, set<string>>& ompThreadPrivate, SgSymbol* toFind)
-{
-    
-    auto it = ompThreadPrivate.find(st);
-    if (it == ompThreadPrivate.end())
-    {
-        it = ompThreadPrivate.insert(it, make_pair(st, set<string>()));
-
-        SgStatement* lastN = st->lastNodeOfStmt();
-        set<string> dummy;
-        do
-        {
-            st = st->lexNext();
-
-            auto res = parseOmpInStatement(st, dummy);
-            for (auto& dir : res)
-                for (auto& var : dir.threadPrivVars)
-                    it->second.insert(var);
-        } while (st != lastN && !isSgExecutableStatement(st) && st->variant() != CONTAINS_STMT);
-    }
-
-    if (it->second.find(toFind->identifier()) != it->second.end())
-        return true;
-    else
-        return false;
-}
-
-static bool hasAssingOpInDecl(SgSymbol* symb)
-{
-    vector<SgStatement*> allDecls;
-    SgStatement* decl = declaratedInStmt(symb, &allDecls);
-    
-    for (auto& elem : allDecls)
-    {
-        if (elem->variant() == VAR_DECL_90)
-        {
-            SgExpression* list = elem->expr(0);
-            while (list)
-            {
-                if (list->lhs()->variant() == ASSGN_OP)
-                    if (list->lhs()->lhs()->symbol() && OriginalSymbol(list->lhs()->lhs()->symbol()) == symb)
-                        return true;
-                list = list->rhs();
-            }
-        }
-    }
-
-    return false;
-}
-
-static set<tuple<int, string, string>> checkedArraysForWrongLocation;
-
-static string getNameWithScope(SgStatement* scope, const string& currFunctionName)
-{
-    if (scope && isSgProgHedrStmt(scope) && scope->symbol()->identifier() != currFunctionName)
-        return scope->symbol()->identifier();
-    else
-        return currFunctionName;
-}
-
-struct findInfo
-{
-    findInfo(const string fName, SgExpression* ex, int parN, bool isWrite) : 
-        fName(fName), ex(ex), parN(parN), isWrite(isWrite) 
-    { }
-
-    SgExpression* ex;
-    string fName;
-    int parN;
-    bool isWrite;
-};
-
-static void findArrayRefs(SgExpression *ex, SgStatement *st, string fName, int parN, bool isWrite,
-                          const map<string, vector<SgExpression*>> &commonBlocks,                          
-                          map<tuple<int, string, string>, pair<DIST::Array*, DIST::ArrayAccessInfo*>> &declaredArrays,
-                          map<SgStatement*, set<tuple<int, string, string>>> &declaratedArraysSt,
-                          const set<string> &privates, const set<string> &deprecatedByIO, 
-                          bool isExecutable, const string &currFunctionName, 
-                          const vector<string> &inRegion,
-                          const set<string> &funcParNames,
-                          map<SgStatement*, set<string>>& ompThreadPrivate, 
-                          const map<string, int>& distrStateFromGUI,
-                          const bool saveAllLocals,
-                          map<string, vector<Messages>>& currMessages,
-                          int& errorCount)
-{
-    const string globalFile = current_file->filename();
-    const set<string> filesInProj = getAllFilesInProject();
-
-    if (ex == NULL)
-        return;
-    stack<findInfo> queue;
-    queue.push(findInfo(fName, ex, parN, isWrite));
-
-    while (!queue.empty())
-    {
-        const findInfo& curQ = queue.top();
-        ex = curQ.ex;
-        fName = curQ.fName;
-        parN = curQ.parN;
-        isWrite = curQ.isWrite;
-
-        queue.pop();
-
-        if (isArrayRef(ex))
-        {
-            SgSymbol* symb = OriginalSymbol(ex->symbol());
-            const bool inDataStat = (symb->attributes() & DATA_BIT) != 0 || ((ex->symbol())->attributes() & DATA_BIT);
-            checkNull(symb->type(), convertFileName(__FILE__).c_str(), __LINE__);
-
-            const int typeSize = getSizeOfType(symb->type()->baseType());
-            if (typeSize == 0)
-            {
-                //__spf_print(1, "Wrong type size for array %s\n", symb->identifier());
-                //printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
-            }
-
-            SgStatement* decl = declaratedInStmt(symb);
-            if (decl->variant() == DVM_VAR_DECL || decl->variant() == HPF_TEMPLATE_STAT)
-            {
-                const string tmp(decl->unparse());
-                if (tmp.find("!DVM$ TEMPLATE") != string::npos)
-                {
-                    auto sTemp = symb->identifier();
-                    tuple<int, string, string> uniqKey;
-                    bool found = false;
-                    for (auto& elem : declaredArrays)
-                    {
-                        if (elem.second.first->GetShortName() == sTemp)
-                        {
-                            uniqKey = elem.first;
-                            found = true;
-                        }
-                    }
-
-                    if (found)
-                    {
-                        auto itDecl = declaratedArraysSt.find(decl);
-                        if (itDecl == declaratedArraysSt.end())
-                            itDecl = declaratedArraysSt.insert(itDecl, make_pair(decl, set<tuple<int, string, string>>()));
-                        itDecl->second.insert(uniqKey);
-
-                        return;
-                    }
-                }
-            }
-
-            auto uniqKey = getUniqName(commonBlocks, decl, symb);
-
-            SgStatement* scope = symb->scope();
-            pair<DIST::arrayLocType, string> arrayLocation;
-
-            string typePrefix = "";
-            while (scope && scope->variant() == STRUCT_DECL)
-            {
-                if (typePrefix == "")
-                    typePrefix = scope->symbol()->identifier();
-                else
-                    typePrefix += scope->symbol()->identifier() + string("::");
-                scope = scope->controlParent();
-            }
-
-            if ((ex->symbol() && symb != ex->symbol()) || (scope && scope->variant() == MODULE_STMT))
-            {
-                if (scope)
-                {
-                    string modName = scope->symbol()->identifier();
-                    arrayLocation = make_pair(DIST::l_MODULE, (typePrefix == "") ? modName : modName + "::" + typePrefix);
-                }
-                else //TODO: find module name with another way
-                    arrayLocation = make_pair(DIST::l_MODULE, "UNREC_MODULE_NAME");
-            }
-            else if (get<1>(uniqKey).find("common_") != string::npos)
-                arrayLocation = make_pair(DIST::l_COMMON, get<1>(uniqKey).substr(strlen("common_")));
-            else if (funcParNames.find(symb->identifier()) != funcParNames.end())
-                arrayLocation = make_pair(DIST::l_PARAMETER, getNameWithScope(scope, currFunctionName));
-            else
-            {
-                if (saveAllLocals || ((symb->attributes() & SAVE_BIT) != 0) || hasAssingOpInDecl(symb))
-                    arrayLocation = make_pair(DIST::l_LOCAL_SAVE, getNameWithScope(scope, currFunctionName));
-                else
-                    arrayLocation = make_pair(DIST::l_LOCAL, getNameWithScope(scope, currFunctionName));
-            }
-
-            auto itNew = declaredArrays.find(uniqKey);
-            if (itNew == declaredArrays.end())
-            {
-                DIST::Array* arrayToAdd =
-                    new DIST::Array(getShortName(uniqKey), symb->identifier(), ((SgArrayType*)(symb->type()))->dimension(),
-                                    getUniqArrayId(), decl->fileName(), decl->lineNumber(), arrayLocation, new Symbol(symb),
-                                    findOmpThreadPrivDecl(scope, ompThreadPrivate, symb), false, false,
-                                    inRegion, typeSize, sharedMemoryParallelization ? DIST::NO_DISTR : DIST::DISTR);
-
-                itNew = declaredArrays.insert(itNew, make_pair(uniqKey, make_pair(arrayToAdd, new DIST::ArrayAccessInfo())));
-
-                vector<pair<int, int>> sizes;
-                map<DIST::Array*, set<DIST::Array*>> arrayLinksByFuncCallsNotReady;
-                map<string, vector<FuncInfo*>> allFuncInfoNoReady;
-                auto sizesExpr = getArraySizes(sizes, symb, decl, arrayLinksByFuncCallsNotReady, allFuncInfoNoReady);
-                arrayToAdd->SetSizes(sizes);
-                arrayToAdd->SetSizesExpr(sizesExpr);
-                tableOfUniqNamesByArray[arrayToAdd] = uniqKey;
-            }
-            else // check the same location from include!
-            {
-                auto prevLocation = itNew->second.first->GetLocation();
-                if (prevLocation != arrayLocation)
-                {
-                    if (checkedArraysForWrongLocation.find(uniqKey) == checkedArraysForWrongLocation.end())
-                    {
-                        checkedArraysForWrongLocation.insert(uniqKey);
-                        __spf_print(1, "can not change declaration area of array '%s' on line %d\n", symb->identifier(), st->lineNumber());
-
-                        wstring messageE, messageR;
-                        __spf_printToLongBuf(messageE, L"Array '%s' has declaration area conflict, it might be worth applying the Include inlining pass", to_wstring(symb->identifier()).c_str());
-                        __spf_printToLongBuf(messageR, R184, to_wstring(symb->identifier()).c_str());
-
-                        currMessages[st->fileName()].push_back(Messages(ERROR, st->lineNumber(), messageR, messageE, 1061));
-                    }
-                    errorCount++;
-                }
-            }
-
-            if ((symb->attributes() & EQUIVALENCE_BIT) != 0)
-                itNew->second.first->SetEquvalence(true);
-
-            for (auto& reg : inRegion)
-                itNew->second.first->SetRegionPlace(reg);
-
-            const auto oldVal = itNew->second.first->GetDistributeFlagVal();
-            if (oldVal == DIST::DISTR || oldVal == DIST::NO_DISTR)
-            {
-                if (itNew->second.first->IsOmpThreadPrivate())
-                    itNew->second.first->SetDistributeFlag(DIST::SPF_PRIV);
-                else if (privates.find(symb->identifier()) != privates.end())
-                {
-                    //check in module
-                    if (itNew->second.first->GetLocation().first == DIST::l_MODULE)
-                    {
-                        for (auto& data : getAttributes<SgStatement*, SgStatement*>(decl, set<int>{ SPF_ANALYSIS_DIR }))
-                        {
-                            set<string> privatesS;
-                            fillPrivatesFromComment(new Statement(data), privatesS);
-                            if (privatesS.find(symb->identifier()) != privatesS.end())
-                            {
-                                itNew->second.first->SetDistributeFlag(DIST::SPF_PRIV);
-                                break;
-                            }
-                        }
-
-                        auto prev = decl->lexPrev();
-                        checkNull(prev, convertFileName(__FILE__).c_str(), __LINE__);
-
-                        set<string> privatesS;
-                        fillPrivatesFromComment(new Statement(prev), privatesS);
-                        if (privatesS.find(symb->identifier()) != privatesS.end())
-                            itNew->second.first->SetDistributeFlag(DIST::SPF_PRIV);
-                    }
-                    else
-                        itNew->second.first->SetDistributeFlag(DIST::SPF_PRIV);
-                }
-                else if (deprecatedByIO.find(symb->identifier()) != deprecatedByIO.end())
-                    itNew->second.first->SetDistributeFlag(DIST::IO_PRIV);
-                else if (isSgConstantSymb(symb) || inDataStat)
-                    itNew->second.first->SetDistributeFlag(DIST::SPF_PRIV);
-                else
-                {
-                    auto it = distrStateFromGUI.find(itNew->second.first->GetIndepUniqName());
-                    if (it != distrStateFromGUI.end())
-                    {
-                        if (it->second != oldVal)
-                        {
-                            itNew->second.first->SetDistributeFlag((DIST::distFlag)it->second);
-                            __spf_print(1, "change flag for array from cache '%s': %d -> %d\n", it->first.c_str(), oldVal, it->second);
-                        }
-                    }
-                    else
-                        itNew->second.first->SetDistributeFlag(DIST::DISTR);
-                }
-            }
-
-            if (typeSize == 0) // unknown
-                itNew->second.first->SetDistributeFlag(DIST::SPF_PRIV);
-
-
-            if (!isExecutable)
-                itNew->second.first->AddDeclInfo(make_pair(st->fileName(), st->lineNumber()), filesInProj, globalFile, new Symbol(symb));
-
-            if (isExecutable)
-            {
-                if (st->variant() != ALLOCATE_STMT && st->variant() != DEALLOCATE_STMT)
-                {
-                    itNew->second.second->AddAccessInfo(st->fileName(), make_pair(st->lineNumber(), isWrite ? 1 : 0), fName, parN);
-                    itNew->second.first->AddUsagePlace(st->fileName(), st->lineNumber());
-                }
-            }
-
-            auto itDecl = declaratedArraysSt.find(decl);
-            if (itDecl == declaratedArraysSt.end())
-                itDecl = declaratedArraysSt.insert(itDecl, make_pair(decl, set<tuple<int, string, string>>()));
-            itDecl->second.insert(uniqKey);
-
-            if (decl->variant() == DVM_VAR_DECL || decl->variant() == HPF_TEMPLATE_STAT)
-            {
-                const string tmp(decl->unparse());
-                if (tmp.find("!DVM$ TEMPLATE") != string::npos)
-                {
-                    itNew->second.first->SetTemplateFlag(true);
-                    //TODO: analyze align mapping
-                    for (int z = 0; z < itNew->second.first->GetDimSize(); ++z)
-                        itNew->second.first->SetMappedDim(z);
-                }
-            }
-        }
-
-        if (ex->variant() == FUNC_CALL)
-        {
-            SgFunctionCallExp* funcExp = (SgFunctionCallExp*)ex;
-            const string fName = funcExp->funName()->identifier();
-            bool intr = (isIntrinsicFunctionName(fName.c_str()) == 1);
-            for (int z = 0; z < funcExp->numberOfArgs(); ++z)
-            {
-                //assume all arguments of function as OUT, except for inctrinsics
-                bool isWriteN = intr ? false : true;
-                //need to correct W/R usage with GraphCall map later
-                findArrayRefs(funcExp->arg(z), st, fName, z, isWriteN, commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO, isExecutable, currFunctionName, inRegion, funcParNames, ompThreadPrivate, distrStateFromGUI, saveAllLocals, currMessages, errorCount);
-            }
-        }
-        else
-        {
-            bool isWriteN = false;
-            if (ex->lhs())
-                queue.push(findInfo("", ex->lhs(), -1, isWriteN));
-            if (ex->rhs())
-                queue.push(findInfo("", ex->rhs(), -1, isWriteN));
-            //findArrayRefs(ex->lhs(), st, "", -1, isWriteN, commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO, isExecutable, currFunctionName, inRegion, funcParNames, ompThreadPrivate, distrStateFromGUI, saveAllLocals, currMessages, errorCount);
-            //findArrayRefs(ex->rhs(), st, "", -1, isWriteN, commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO, isExecutable, currFunctionName, inRegion, funcParNames, ompThreadPrivate, distrStateFromGUI, saveAllLocals, currMessages, errorCount);
-        }
-    }
-}
-
-static void findArrayRefInIO(SgExpression *ex, set<string> &deprecatedByIO, SgStatement *st, map<string, vector<Messages>> &currMessages)
-{
-    if (ex)
-    {
-        if (ex->variant() == ARRAY_REF)
-        {
-            auto symb = ex->symbol();
-            if (symb->type())
-            {
-                if (symb->type()->variant() == T_ARRAY)
-                {
-                    auto found = deprecatedByIO.find(OriginalSymbol(symb)->identifier());
-                    if (found == deprecatedByIO.end())
-                    {
-                        deprecatedByIO.insert(found, OriginalSymbol(symb)->identifier());
-
-                        wstring messageE, messageR;
-                        __spf_printToLongBuf(messageE, L"Array '%s' can not be distributed because of DVM's I/O constraints", to_wstring(symb->identifier()).c_str());
-                        __spf_printToLongBuf(messageR, R68, to_wstring(symb->identifier()).c_str());
-
-                        currMessages[st->fileName()].push_back(Messages(WARR, st->lineNumber(), messageR, messageE, 1037));
-
-                        __spf_print(1, "Array '%s' at line %d can not be distributed because of DVM's I/O constraints\n", symb->identifier(), st->lineNumber());
-                    }
-                }
-            }
-        }
-        
-        findArrayRefInIO(ex->lhs(), deprecatedByIO, st, currMessages);
-        findArrayRefInIO(ex->rhs(), deprecatedByIO, st, currMessages);
-    }
-}
-
-static void findReshape(SgStatement *st, set<string> &privates, map<string, vector<Messages>> &currMessages)
-{
-    if (st->variant() == ASSIGN_STAT)
-    {
-        SgExpression *exL = st->expr(0);
-        SgExpression *exR = st->expr(1);
-
-        if (exR->variant() == FUNC_CALL && exL->variant() == ARRAY_REF)
-        {
-            if (exR->symbol()->identifier() == string("reshape"))
-            {
-                if (privates.find(exL->symbol()->identifier()) == privates.end())
-                {
-                    privates.insert(exL->symbol()->identifier());
-
-                    wstring messageE, messageR;
-                    __spf_printToLongBuf(messageE, L"Array '%s' can not be distributed because of RESHAPE", to_wstring(exL->symbol()->identifier()).c_str());
-                    __spf_printToLongBuf(messageR, R90, to_wstring(exL->symbol()->identifier()).c_str());
-                    currMessages[st->fileName()].push_back(Messages(NOTE, st->lineNumber(), messageR, messageE, 1047));
-                }
-            }
-        }
-    }
-}
-
-static void findConstructorRef(SgStatement* st, set<string>& privates, map<string, vector<Messages>>& currMessages)
-{
-    if (st->variant() == ASSIGN_STAT)
-    {
-        SgExpression* exL = st->expr(0);
-        SgExpression* exR = st->expr(1);
-
-        if (exR->variant() == CONSTRUCTOR_REF && exL->variant() == ARRAY_REF)
-        {
-            if (privates.find(exL->symbol()->identifier()) == privates.end())
-            {
-                privates.insert(exL->symbol()->identifier());
-
-                wstring messageE, messageR;
-                __spf_printToLongBuf(messageE, L"Array '%s' can not be distributed because of initializer list", to_wstring(exL->symbol()->identifier()).c_str());
-                __spf_printToLongBuf(messageR, R164, to_wstring(exL->symbol()->identifier()).c_str());
-                currMessages[st->fileName()].push_back(Messages(NOTE, st->lineNumber(), messageR, messageE, 1047));
-            }
-        }
-    }
-}
-
-static void addPrivates(SgStatement *st, set<string>& privates, map<string, set<Symbol*>>& reductions, 
-                        map<string, set<tuple<Symbol*, Symbol*, int>>>& reductionsLoc)
-{
-    //after SPF preprocessing 
-    for (auto& data : getAttributes<SgStatement*, SgStatement*>(st, set<int>{ SPF_ANALYSIS_DIR }))
-    {
-        set<Symbol*> privatesS;
-        fillPrivatesFromComment(new Statement(data), privatesS);
-        fillReductionsFromComment(new Statement(data), reductions);
-        fillReductionsFromComment(new Statement(data), reductionsLoc);
-
-        for (auto& elem : privatesS)
-            privates.insert(elem->GetOriginal()->identifier());
-    }
-
-    //before SPF preprocessing 
-    if (st->variant() == SPF_ANALYSIS_DIR)
-    {
-        set<Symbol*> privatesS;
-        fillPrivatesFromComment(new Statement(st), privatesS);
-        fillReductionsFromComment(new Statement(st), reductions);
-        fillReductionsFromComment(new Statement(st), reductionsLoc);
-
-        for (auto& elem : privatesS)
-            privates.insert(elem->GetOriginal()->identifier());
-    }
-}
-
-int getAllDeclaredArrays(SgFile *file, map<tuple<int, string, string>, pair<DIST::Array*, DIST::ArrayAccessInfo*>> &declaredArrays,
-                         map<SgStatement*, set<tuple<int, string, string>>> &declaratedArraysSt, map<string, vector<Messages>> &currMessages,
-                         const vector<ParallelRegion*> &regions, const map<string, int>& distrStateFromGUI)
-{
-    int countErrors = 0;
-
-    vector<SgStatement*> modules;
-    findModulesInFile(file, modules);
-
-    map<string, set<string>> privatesByModule;
-    for (auto &mod : modules)
-    {
-        const string modName = mod->symbol()->identifier();
-        privatesByModule[modName] = set<string>();
-        auto it = privatesByModule.find(modName);
-
-        for (SgStatement *iter = mod; iter != mod->lastNodeOfStmt(); iter = iter->lexNext())
-        {
-            if (iter->variant() == CONTAINS_STMT)
-                break;
-
-            //after SPF preprocessing 
-            for (auto &data : getAttributes<SgStatement*, SgStatement*>(iter, set<int>{ SPF_ANALYSIS_DIR }))
-                fillPrivatesFromComment(new Statement(data), it->second);
-
-            //before SPF preprocessing 
-            if (iter->variant() == SPF_ANALYSIS_DIR)
-                fillPrivatesFromComment(new Statement(iter), it->second);
-        }
-    }
-
-    map<SgStatement*, set<string>> ompThreadPrivate;
-
-    for (int i = 0; i < file->numberOfFunctions(); ++i)
-    {
-        bool saveAllLocals = false;
-
-        SgStatement *st = file->functions(i);
-        SgStatement *lastNode = st->lastNodeOfStmt();
-        map<string, vector<SgExpression*>> commonBlocks;
-        const string currFunctionName = st->symbol()->identifier();
-
-        getCommonBlocksRef(commonBlocks, st, lastNode);
-        set<string> privates;
-        set<string> deprecatedByIO;
-        map<string, set<Symbol*>> reductions;
-        map<string, set<tuple<Symbol*, Symbol*, int>>> reductionsLoc;
-        set<string> funcParNames;
-
-        if (st->variant() != PROG_HEDR)
-        {
-            SgProcHedrStmt *func = (SgProcHedrStmt*)st;
-            for (int z = 0; z < func->numberOfParameters(); ++z)
-                funcParNames.insert(func->parameter(z)->identifier());
-
-            if (func->nameWithContains() != func->name().identifier()) // added contains args
-            {
-                SgProcHedrStmt* cp = (SgProcHedrStmt*)func->controlParent();
-                checkNull(cp, convertFileName(__FILE__).c_str(), __LINE__);
-
-                for (int z = 0; z < cp->numberOfParameters(); ++z)
-                    funcParNames.insert(cp->parameter(z)->identifier());
-            }
-        }
-
-        for (SgStatement* iter = st; iter != lastNode; iter = iter->lexNext())
-        {
-            if (iter->variant() == CONTAINS_STMT)
-                break;
-
-            addPrivates(iter, privates, reductions, reductionsLoc);
-
-            if (iter->variant() == USE_STMT)
-                fillFromModule(iter->symbol(), privatesByModule, privates);
-
-            if (iter->variant() == SAVE_DECL)
-                if (!iter->expr(0) && !iter->expr(1) && !iter->expr(2))
-                    saveAllLocals = true;
-        }
-
-        for (SgStatement* iter = st; iter != lastNode; iter = iter->lexNext())
-        {
-            if (iter->variant() == CONTAINS_STMT)
-                break;
-
-            findReshape(iter, privates, currMessages);
-            findConstructorRef(iter, privates, currMessages);            
-        }
-
-        SgStatement* tmpModFind = st;
-        while (tmpModFind->variant() != GLOBAL)
-        {
-            tmpModFind = tmpModFind->controlParent();
-            if (tmpModFind->variant() == MODULE_STMT)
-                fillFromModule(tmpModFind->symbol(), privatesByModule, privates);
-        }
-
-        SgStatement *currF = st;
-        SgStatement *contains = isSgProgHedrStmt(currF->controlParent());
-        if (contains)
-        {
-            for (SgStatement *loc = contains; loc; loc = loc->lexNext())
-            {
-                if (isSgExecutableStatement(loc))
-                    break;
-                if (loc->variant() == CONTAINS_STMT)
-                    break;
-
-                if (loc->variant() == USE_STMT)
-                    fillFromModule(loc->symbol(), privatesByModule, privates);
-            }
-        }
-
-        for (auto &elem : reductions)
-            for (auto &setElem : elem.second)
-                privates.insert(setElem->identifier());
-
-        for (auto &elem : reductionsLoc)
-        {
-            for (auto &setElem : elem.second)
-            {
-                privates.insert(get<0>(setElem)->identifier());
-                privates.insert(get<1>(setElem)->identifier());
-            }
-        }
-
-        //analyze IO operations
-        if (!ignoreIO)
-        {
-            for (SgStatement* iter = st; iter != lastNode; iter = iter->lexNext())
-            {
-                if (iter->variant() == CONTAINS_STMT)
-                    break;
-
-                SgInputOutputStmt* stIO = isSgInputOutputStmt(iter);
-                set<ParallelRegion*> currRegs = getAllRegionsByLine(regions, iter->fileName(), iter->lineNumber());
-                if (stIO && currRegs.size()) // deprecate to distribute arrays only in regions
-                {
-                    int countOfItems = 0;
-                    for (SgExpression* items = stIO->itemList(); items; items = items->rhs(), ++countOfItems);
-
-                    //TODO: need to add more checkers!
-                    if (countOfItems > 1)
-                    {
-                        for (SgExpression* items = stIO->itemList(); items; items = items->rhs())
-                            findArrayRefInIO(items->lhs(), deprecatedByIO, stIO, currMessages);
-                    }
-                    else if (countOfItems == 1)
-                    {
-                        auto list = stIO->specList();
-                        bool ok = true;
-                        //exclude FMT='format'
-                        while (list)
-                        {
-                            if (list->lhs() && list->lhs()->variant() == SPEC_PAIR)
-                            {
-                                auto ex = list->lhs();
-                                if (ex->lhs() && ex->rhs())
-                                {
-                                    if (ex->lhs()->variant() == KEYWORD_VAL)
-                                    {
-                                        SgKeywordValExp* key = (SgKeywordValExp*)(ex->lhs());
-                                        if (key->value() == string("fmt"))
-                                            if (ex->rhs()->variant() == STRING_VAL)
-                                                ok = false;
-                                    }
-                                }
-                            }
-
-                            if (!ok)
-                                break;
-                            list = list->rhs();
-                        }
-
-                        //check A(i,j) for example
-                        auto item = stIO->itemList()->lhs();
-                        if (item->rhs() || item->lhs())
-                            ok = false;
-
-                        if (!ok)
-                            findArrayRefInIO(item, deprecatedByIO, stIO, currMessages);
-                    }
-                }
-            }
-        }
-                
-        while (st != lastNode)
-        {
-            if (st->variant() == CONTAINS_STMT)
-                break;            
-
-            if (!isSPF_stat(st) && !isDVM_stat(st))
-            {
-                set<ParallelRegion*> currRegs = getAllRegionsByLine(regions, st->fileName(), st->lineNumber());
-                vector<string> regNames;
-                for (auto &reg : currRegs)
-                    regNames.push_back(reg->GetName());
-                if (regNames.size() == 0)
-                    regNames.push_back("default");
-                
-                if (st->variant() == PROC_STAT)
-                {
-                    SgCallStmt* funcExp = (SgCallStmt*)st;
-                    const string fName = funcExp->symbol()->identifier();
-                    for (int z = 0; z < funcExp->numberOfArgs(); ++z)
-                    {
-                        findArrayRefs(funcExp->arg(z), st, fName, z, true,
-                                      commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO,
-                                      isSgExecutableStatement(st) ? true : false, currFunctionName,
-                                      regNames, funcParNames, ompThreadPrivate, distrStateFromGUI, saveAllLocals,
-                                      currMessages, countErrors);
-                    }
-                }
-                else
-                {
-                    for (int i = 0; i < 3; ++i)
-                        findArrayRefs(st->expr(i), st, "", -1, (st->variant() == ASSIGN_STAT && i == 0) ? true : false,
-                            commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO,
-                                      isSgExecutableStatement(st) ? true : false, currFunctionName,
-                                      regNames, funcParNames, ompThreadPrivate, distrStateFromGUI, saveAllLocals,
-                                      currMessages, countErrors);
-                }
-            }
-            st = st->lexNext();
-        }
-    }
-
-    //preprocess only module declaration 
-    for (auto &mod : modules)
-    {
-        SgStatement *st = mod->lexNext();
-        SgStatement *lastNode = mod->lastNodeOfStmt();
-        map<string, vector<SgExpression*>> commonBlocks;
-        set<string> privates;
-        set<string> deprecatedByIO;
-        set<string> funcParNames;
-
-        fillFromModule(st->symbol(), privatesByModule, privates);
-        
-        while (st != lastNode)
-        {
-            if (st->variant() == CONTAINS_STMT)
-                break;
-
-            if (!isSPF_stat(st) && !isDVM_stat(st))
-            {
-                //TODO: set clear regions for modules
-                set<ParallelRegion*> currRegs = getAllRegionsByLine(regions, st->fileName(), st->lineNumber());
-                vector<string> regNames;
-                for (auto& reg : currRegs)
-                    regNames.push_back(reg->GetName());
-                if (regNames.size() == 0)
-                    regNames.push_back("default");
-
-                for (int i = 0; i < 3; ++i)
-                    findArrayRefs(st->expr(i), st, "", -1, false, commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO,
-                                  false, "NULL", regNames, funcParNames, ompThreadPrivate, distrStateFromGUI, false,
-                                  currMessages, countErrors);
-            }
-            st = st->lexNext();
-        }
-    }
-
-    //preprocess only block data declaration 
-    for (SgStatement* st = file->firstStatement()->lexNext(); st; st = st->lastNodeOfStmt(), st = st->lexNext())
-    {
-        if (st->variant() == BLOCK_DATA)
-        {            
-            SgStatement* last = st->lastNodeOfStmt();
-            SgStatement* curr = st;
-
-            map<string, vector<SgExpression*>> commonBlocks;
-            getCommonBlocksRef(commonBlocks, st, last);
-
-            set<string> privates;
-            set<string> deprecatedByIO;
-            set<string> funcParNames;
-
-            string blockName = "BLOCK DATA";
-            if (st->symbol())
-                blockName = st->symbol()->identifier();
-
-            while (curr && curr != last)
-            {
-                //TODO: set clear regions for block data
-                set<ParallelRegion*> currRegs = getAllRegionsByLine(regions, curr->fileName(), curr->lineNumber());
-                vector<string> regNames;
-                for (auto& reg : currRegs)
-                    regNames.push_back(reg->GetName());
-                if (regNames.size() == 0)
-                    regNames.push_back("default");
-
-                for (int i = 0; i < 3; ++i)
-                    findArrayRefs(curr->expr(i), curr, "", -1, false, commonBlocks, declaredArrays, declaratedArraysSt, privates, deprecatedByIO,
-                                  false, blockName, regNames, funcParNames, ompThreadPrivate, distrStateFromGUI, false,
-                                  currMessages, countErrors);
-                curr = curr->lexNext();
-            }
-        }
-    }
-    return countErrors;
-}
-
 void insertSpfAnalysisBeforeParalleLoops(const vector<LoopGraph*> &loops)
 {
     for (auto &loop : loops)

sapfor/experts/Sapfor_2017/_src/LoopAnalyzer/loop_analyzer.h

@@ -33,6 +33,13 @@ void loopAnalyzer(SgFile *file,
                   const std::map<DIST::Array*, std::set<DIST::Array*>> &arrayLinksByFuncCalls, 
                   const std::map<SgStatement*, std::vector<DefUseList>> &defUseByPlace,
                   bool skipDeps, std::vector<LoopGraph*> *loopGraph = NULL);
+
+void fillFromModule(SgSymbol* s, const std::map<std::string, std::set<std::string>>& privatesByModule, std::set<std::string>& privates);
+
+std::vector<std::pair<Expression*, Expression*>> getArraySizes(std::vector<std::pair<int, int>>& sizes, SgSymbol* symb, SgStatement* decl,
+                                                               const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls,
+                                                               const std::map<std::string, std::vector<FuncInfo*>>& allFuncInfo);
+
 void arrayAccessAnalyzer(SgFile *file, std::vector<Messages> &messagesForFile, 
                          const std::map<std::tuple<int, std::string, std::string>, std::pair<DIST::Array*, DIST::ArrayAccessInfo*>> &declaredArrays, 
                          REGIME regime);
@@ -46,9 +53,6 @@ bool isIntrinsic(const char *funName);
 std::tuple<int, std::string, std::string> getUniqName(const std::map<std::string, SgStatement*> &commonBlocks, SgStatement *decl, SgSymbol *symb);
 std::string getShortName(const std::tuple<int, std::string, std::string> &uniqKey);
 
-int getAllDeclaredArrays(SgFile *file, std::map<std::tuple<int, std::string, std::string>, std::pair<DIST::Array*, DIST::ArrayAccessInfo*>> &declaredArrays,
-                         std::map<SgStatement*, std::set<std::tuple<int, std::string, std::string>>> &declaratedArraysSt, std::map<std::string, std::vector<Messages>> &currMessages,
-                         const std::vector<ParallelRegion*> &regions, const std::map<std::string, int>& distrStateFromGUI);
 void insertSpfAnalysisBeforeParalleLoops(const std::vector<LoopGraph*> &loops);
 void recalculateArraySizes(std::set<DIST::Array*> &arraysDone, const std::set<DIST::Array*> &allArrays, const std::map<DIST::Array*, std::set<DIST::Array*>> &arrayLinksByFuncCalls, const std::map<std::string, std::vector<FuncInfo*>>& allFuncInfo);
 int getSizeOfType(SgType* t);
@@ -93,3 +97,8 @@ template<int NUM> bool createRemoteDir(SgStatement *st, const std::map<std::stri
 void groupActualAndRemote(SgFile* file, bool revert = false);
 
 std::vector<SgStatement*> filterUserSpf(const std::vector<SgStatement*>& toFilter, bool with_omp = true);
+
+// ArrayAnalysis.cpp
+int getAllDeclaredArrays(SgFile *file, std::map<std::tuple<int, std::string, std::string>, std::pair<DIST::Array*, DIST::ArrayAccessInfo*>> &declaredArrays,
+                         std::map<SgStatement*, std::set<std::tuple<int, std::string, std::string>>> &declaratedArraysSt, std::map<std::string, std::vector<Messages>> &currMessages,
+                         const std::vector<ParallelRegion*> &regions, const std::map<std::string, int>& distrStateFromGUI);

sapfor/experts/Sapfor_2017/_src/ParallelizationRegions/resolve_par_reg_conflicts.cpp

@@ -958,13 +958,13 @@ static bool replaceCommonArray(const string &fileName,
     return false;
 }
 
-static pair<SgSymbol*, SgSymbol*> copyArray(const pair<string, int> &place,
-                                            const DIST::Array *array,
-                                            const vector<ParallelRegionLines>& lines,
-                                            const string &suffix, 
-                                            string& filename, 
-                                            map<string, map<int, set<string>>>& newDeclsToInclude,
-                                            map<string, map<int, set<string>>>& copied)
+pair<SgSymbol*, SgSymbol*> copyArray(const pair<string, int> &place,
+                                     const DIST::Array *array,
+                                     const vector<ParallelRegionLines>& lines,
+                                     const string &suffix, 
+                                     string& filename, 
+                                     map<string, map<int, set<string>>>& newDeclsToInclude,
+                                     map<string, map<int, set<string>>>& copied)
 {
     string fileName = place.first;    
     int switchRes = SgFile::switchToFile(fileName);

sapfor/experts/Sapfor_2017/_src/ParallelizationRegions/resolve_par_reg_conflicts.h

@@ -13,4 +13,12 @@ int printCheckRegions(const char *fileName, const std::vector<ParallelRegion*> &
 
 bool checkRegionsResolving(const std::vector<ParallelRegion*> &regions, const std::map<std::string, std::vector<FuncInfo*>> &allFuncInfo, const std::map<std::string, CommonBlock*> &commonBlocks, std::map<std::string, std::vector<Messages>> &SPF_messages, bool sharedMemoryParallelization);
 int resolveParRegions(std::vector<ParallelRegion*>& regions, const std::map<std::string, std::vector<FuncInfo*>>& allFuncInfo, std::map<std::string, std::vector<Messages>>& SPF_messages, bool sharedMemoryParallelization, std::map<std::string, std::map<int, std::set<std::string>>>& copyDecls);
-void insertRealignsBeforeFragments(ParallelRegion* reg, SgFile* file, const std::set<DIST::Array*>& distrArrays, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls);
+void insertRealignsBeforeFragments(ParallelRegion* reg, SgFile* file, const std::set<DIST::Array*>& distrArrays, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls);
+
+std::pair<SgSymbol*, SgSymbol*> copyArray(const std::pair<std::string, int>& place,
+                                          const DIST::Array* array,
+                                          const std::vector<ParallelRegionLines>& lines,
+                                          const std::string& suffix,
+                                          std::string& filename,
+                                          std::map<std::string, std::map<int, std::set<std::string>>>& newDeclsToInclude,
+                                          std::map<std::string, std::map<int, std::set<std::string>>>& copied);

sapfor/experts/Sapfor_2017/_src/Predictor/PredictScheme.cpp

@@ -18,12 +18,23 @@
 #include "../Utils/SgUtils.h"
 #include "../DirectiveProcessing/directive_parser.h"
 #include "../Distribution/DvmhDirective.h"
+#include "../GraphLoop/graph_loops_func.h"
+#include "../ExpressionTransform/expr_transform.h"
+#include "../DirectiveProcessing/directive_parser.h"
+#include "../LoopAnalyzer/loop_analyzer.h"
+#include "../CFGraph/CFGraph.h"
+
+#include "json.hpp"
 
 using std::map;
 using std::string;
 using std::vector;
 using std::set;
 using std::ofstream;
+using std::pair;
+using std::tuple;
+
+using json = nlohmann::json;
 
 static void fillParallel(SgExpression *exp, ParallelStats &parStats, int &totalScoreComm)
 {
@@ -152,8 +163,25 @@ static void calculateForParallelLoop(SgStatement* loop, const map<int, Gcov_info
     }
 }
 
+static json info;
+
 void calculateStatsForPredictor(const map<string, vector<FuncInfo*>>& allFuncInfo, 
                                 const map<string, map<int, Gcov_info>>& gCovInfo) {
+
+    json cluster;
+    json program;
+    
+    cluster["cluster_info"] = { {"num_nodes", 0},
+                                {"cores_per_node", 0},
+                                {"threads_per_node", 0},
+                                {"memory_per_node_gb", 0},
+                                {"network_bandwidth_gbps", 0},
+                                {"network_latency_ms", 0}
+                              };
+        
+    program["program_info"]["sequential_execution_time_sec"] = 0.0;
+    program["program_info"]["launch_grid"] = { {"dimensions", {0, 0, 0} }, {"total_processes", 0} };
+    
     uint64_t total_exec_count = 0;
     uint64_t parallel_exec_count = 0;
     uint64_t count_of_parallel_lines = 0;
@@ -226,17 +254,321 @@ void calculateStatsForPredictor(const map<string, vector<FuncInfo*>>& allFuncInf
     __spf_print(1, "    average_parallel_exec %.16e\n", parallel_exec_count / (double)count_of_parallel_lines);
     __spf_print(1, "    parallel_rate %.16e\n", parallel_exec_count / (double)total_exec_count);
 
-    ofstream stats("stats.csv");
-    stats << "average_parallel_exec;" << parallel_exec_count / (double)count_of_parallel_lines << std::endl;
-    stats << "parallel_rate;" << parallel_exec_count / (double)total_exec_count << std::endl;
-    stats.close();
+    program["program_info"]["average_parallel_line_executions"] = parallel_exec_count / (double)count_of_parallel_lines;
+    program["program_info"]["parallel_execution_fraction"] = parallel_exec_count / (double)total_exec_count;
+
+    info = { cluster, program };
 }
 
-void parseDvmDirForPredictor(const map<string, vector<FuncInfo*>>& allFuncInfo,
+static const Gcov_info& getInfo(SgStatement* st, const map<int, Gcov_info> &gcov)
+{
+    auto stat = st;
+    while (isDVM_stat(stat))
+        stat = stat->lexPrev();
+    int line = stat->lineNumber(); // XXX
+
+    auto list = st->expr(1);
+    auto it = gcov.find(line);
+    auto& info = it->second;
+    if (info.getNumLine() != line)
+    {
+        __spf_print(1, "bad gcov info\n");
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+    }
+
+    return info;
+}
+
+static json parseDistribution(const map<DIST::Array*, int>& byPos, SgSymbol* arr, SgExpression* list, int line)
+{
+    json dist;
+    auto array = getArrayFromDeclarated(declaratedInStmt(arr), arr->identifier());
+    if (array == NULL || byPos.find(array) == byPos.end())
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+    
+    dist["line"] = line;
+    dist["array_id"] = byPos.at(array);
+
+    while (list)
+    {
+        dist["distribution_spec"].push_back(list->lhs()->unparse());
+        list = list->rhs();
+    }
+    return dist;
+}
+
+static json parseAlign(const map<DIST::Array*, int>& byPos, SgSymbol* srcArr, SgSymbol* tgtArr,
+                       SgExpression *listSrc, SgExpression* listTgt, int line)
+{
+    json align;
+
+    auto arraySrc = getArrayFromDeclarated(declaratedInStmt(srcArr), srcArr->identifier());
+    if (arraySrc == NULL || byPos.find(arraySrc) == byPos.end())
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+    auto arrayTgt = getArrayFromDeclarated(declaratedInStmt(tgtArr), tgtArr->identifier());
+    if (arrayTgt == NULL || byPos.find(arrayTgt) == byPos.end())
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+    align["line"] = line;
+    align["source_array_id"] = byPos.at(arraySrc);
+    align["target_array_id"] = byPos.at(arrayTgt);
+
+    vector<pair<string, SgSymbol*>> srcSymbs;
+    auto list = listSrc;
+    while (list)
+    {
+        srcSymbs.push_back({ list->lhs()->unparse(), list->lhs()->symbol() });
+        list = list->rhs();
+    }
+
+    vector<pair<int, int>> coefs(srcSymbs.size());
+    list = listTgt;
+    while (list)
+    {
+        auto exp = list->lhs();
+        bool has = false;
+        for (int z = 0; z < srcSymbs.size(); ++z)
+        {
+            has = recSymbolFind(exp, srcSymbs[z].first, VAR_REF);
+            if (has)
+            {
+                getCoefsOfSubscript(coefs[z], exp, srcSymbs[z].second);
+                if (coefs[z].first == 0)
+                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                break;
+            }
+        }
+        list = list->rhs();
+    }
+
+    for (int z = 0; z < coefs.size(); ++z)
+    {
+        if (coefs[z].first == 0)
+            continue;
+        if (coefs[z].second)
+            align["rules"].push_back({ z, coefs[z].first });
+        else
+            align["rules"].push_back({ z, coefs[z].first, coefs[z].second });
+    }
+    return align;
+}
+
+static SgStatement* findBefore(SgStatement* st)
+{
+    while (st)
+    {
+        st = st->lexPrev();
+
+        if (isSgProgHedrStmt(st))
+            break;
+        if (isDVM_stat(st) || isSPF_stat(st))
+            continue;
+        if (isSgExecutableStatement(st))
+            break;        
+    }
+    return st;
+}
+
+static void fillAcrossShadow(vector<pair<pair<Symbol*, string>, vector<pair<int, int>>>>& dirs, SgStatement *st,
+                             const map<DIST::Array*, int>& byPos, const string& type, json& typed, json& parallel)
+{
+    for (auto& dir : dirs)
+    {
+        auto& symb = dir.first;
+        auto& access = dir.second;
+
+        DIST::Array* arr = getArrayFromDeclarated(declaratedInStmt(symb.first), symb.first->identifier());
+        if (arr == NULL || byPos.find(arr) == byPos.end())
+            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+        json item;
+        item["line"] = st->lineNumber();
+        item["array_id"] = byPos.at(arr);
+        item["communication_pattern"] = "NEAREST_NEIGHBOR";
+
+        if (access.size())
+        {
+            for (int z = 0; z < access.size(); ++z)
+                item["width"].push_back({ z, access[z].first, access[z].second });
+        }
+        else
+        {
+            auto& spec = arr->GetShadowSpec();
+            //TODO: analyze spec of array for shadow
+            for (int z = 0; z < spec.size(); ++z)
+                item["width"].push_back({ z, 1, 1 });
+        }
+
+        typed.push_back(item);
+        parallel["shadow_renews"].push_back(typed.size() - 1);
+    }
+}
+
+static void parallelDir(const map<DIST::Array*, int>& byPos, SgExpression* spec, SgSymbol* arr, SgExpression* arrSpec, 
+                        SgStatement* st, SgExpression* clauses, const map<int, Gcov_info>& gcov, json& directives,
+                        const map<string, CommonBlock*>& commonBlocks, const map<string, vector<FuncInfo*>>& allFuncInfo)
+{
+    json parallel;
+
+    json& shadow_renew = directives["shadow_renew"];
+    json& reduction = directives["reduction"];
+    json& remote_access = directives["remote_access"];
+    json& across = directives["across"];
+
+    vector<pair<string, SgSymbol*>> loopSymbs;
+    auto list = spec;
+    while (list)
+    {
+        loopSymbs.push_back({ list->lhs()->unparse(), list->lhs()->symbol() });
+        list = list->rhs();
+    }
+
+    parallel["line"] = st->lineNumber();
+    parallel["loops_count"] = loopSymbs.size();
+
+    SgStatement* loop = isSgForStmt(st->lexNext());
+    if (loop == NULL)
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+    SgStatement* lastNode = loop->lastNodeOfStmt();
+
+    SgStatement* before = findBefore(loop);
+    if (before == NULL)
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+    vector<int64_t> execs;
+    for (int z = 0; z < loopSymbs.size(); ++z)
+    {
+        auto& info = getInfo(loop, gcov);
+        execs.push_back(info.getExecutedCount());
+        loop = loop->lexNext();
+    }
+
+    for (int z = execs.size() - 1; z > 0; --z)
+        execs[z] /= execs[z - 1];
+
+    auto& info = getInfo(before, gcov);
+    execs[0] /= info.getExecutedCount();
+
+    parallel["iterations_count"] = execs;
+
+    DvmDirective directive;
+    fillInfoFromDirective(new Statement(st), directive);
+
+    vector<int> empty;
+    parallel["shadow_renews"] = empty;
+    parallel["reductions"] = empty;
+    parallel["remote_accesses"] = empty;
+    parallel["acrosses"] = empty;
+       
+    for (auto& op : directive.reduction)
+    {
+        for (auto& var : op.second)
+        {
+            json item;
+            item["line"] = st->lineNumber();
+            item["operation"] = op.first;
+            if (!isSgArrayType(var->type()))
+            {
+                item["reduction_type"] = "SCALAR";
+                item["size_bytes"] = getSizeOfType(var->type());
+                item["elements_count"] = 1;
+            }
+            else
+            {
+                item["reduction_type"] = "ARRAY";
+                auto type = isSgArrayType(var->type());
+                item["size_bytes"] = getSizeOfType(type->baseType());
+                item["elements_count"] = type->dimension();
+            }
+
+            reduction.push_back(item);
+            parallel["reductions"].push_back(reduction.size() - 1);
+        }
+    }
+
+    fillAcrossShadow(directive.shadowRenew, st, byPos, "shadow_renews", shadow_renew, parallel);
+    fillAcrossShadow(directive.across, st, byPos, "acrosses", across, parallel);
+        
+    auto func = getFuncStat(st);
+    auto& funcInFile = allFuncInfo.at(st->fileName());
+    FuncInfo* currF = NULL;
+    for (auto& elem : funcInFile)
+        if (elem->funcName == func->symbol()->identifier())
+            currF = elem;
+
+    if (currF == NULL)
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+    auto cfg = buildCFGforCurrentFunc(func, SAPFOR::CFG_Settings(true, false, false, true, false, false, true), commonBlocks, allFuncInfo);
+    if (cfg.size() != 1)
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+    //TODO IP analysis
+        
+    unsigned countOfAccess = 0;
+    unsigned countOfOps = 0;
+
+    if (cfg.find(currF) == cfg.end())
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+    //skip all parallel loops
+    loop = st->lexNext();
+    for (int z = 0; z < loopSymbs.size(); ++z)
+        loop = loop->lexNext();
+
+    int lineStart = loop->lineNumber();
+    int lineEnd = lastNode->lexNext()->lineNumber();
+
+    //dumpCFG(cfg, false);
+    //TODO: calculate access in bytes
+    for (auto& block : cfg[currF])
+    {
+        for (auto& ir : block->getInstructions())
+        {
+            auto line = ir->getLine();
+            if (line < lineStart || line >= lineEnd)
+                continue;
+
+            auto inst = ir->getInstruction();
+            if (inst->isAccess())
+                countOfAccess++;
+            if (inst->isArith())
+                countOfOps++;
+            //printf("%s %d %d\n", inst->dump().c_str(), inst->isAccess(), inst->isArith());
+        }
+    }    
+    deleteCFG(cfg);
+
+    parallel["computational_intensity"] = countOfOps > 0 ? ((double)countOfOps / (double)countOfAccess) : 0;
+
+    directives["parallel"].push_back(parallel);
+}
+
+void parseDvmDirForPredictor(const map<tuple<int, string, string>, pair<DIST::Array*, DIST::ArrayAccessInfo*>>& declaredArrays,
+                             const map<string, CommonBlock*>& commonBlocks,
+                             const map<string, vector<FuncInfo*>>& allFuncInfo,
                              const map<string, map<int, Gcov_info>>& gCovInfo)
 {
-    ofstream dirs("dirs.csv");
+    auto& program = info[1]["program_info"];
 
+    map<DIST::Array*, int> byPos;
+    int pos = 0;
+    for (auto& arrayElem : declaredArrays)
+    {
+        json jArray;
+        auto& array = arrayElem.second.first;
+        auto sizes = array->GetSizes();
+        for (int z = 0; z < array->GetDimSize(); ++z)
+            jArray["dimensions"].push_back(sizes[z].second - sizes[z].first + 1);
+
+        jArray["name"] = array->GetName();
+        jArray["element_size_bytes"] = array->GetTypeSize();
+
+        program["arrays_info"].push_back(jArray);
+        byPos[array] = pos++;
+    }
+    
+    auto& directives = program["directives"];
     for (auto& byFile : allFuncInfo) 
     {
         int ok = SgFile::switchToFile(byFile.first);
@@ -260,48 +592,43 @@ void parseDvmDirForPredictor(const map<string, vector<FuncInfo*>>& allFuncInfo,
                 SgExpression* list;
                 SgExpression* dup;
                 auto line = 0;
-                
+
                 switch (st->variant())
                 {
                 case DVM_PARALLEL_ON_DIR:
-                    {
-                        auto stat = st;
-                        while (isDVM_stat(stat))
-                            stat = stat->lexPrev();
-                        line = stat->lineNumber(); // XXX
-
-                        list = st->expr(1);
-                        auto it = gcov.find(line);
-                        auto& info = it->second;
-                        if (info.getNumLine() != line)
-                        {
-                            __spf_print(1, "bad gcov info\n");
-                            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
-                        }
-
-                        dirs << info.getExecutedCount() << ";" << "PARALLEL;" << st->expr(2)->unparse() << ";" << st->expr(0)->unparse() << ";";
-                        while (list)
-                        {
-                            dirs << list->lhs()->unparse() << ";";
-                            list = list->rhs();
-                        }
-
-                        dirs << std::endl;
-                    }
+                    parallelDir(byPos, st->expr(2), st->expr(0)->symbol(), st->expr(0)->lhs(), st, st->expr(1), gcov, directives, commonBlocks, allFuncInfo);
                     break;
                 case DVM_VAR_DECL: // TODO
-                    dup = st->expr(2)->lhs()->copyPtr();
-                    dup->setLhs(NULL);
-                    dirs << "1;" << dup->unparse() << ";" << st->expr(2)->lhs()->lhs()->unparse() << ";" << st->expr(0)->unparse() << ";\n";
+                {
+                    auto type = st->expr(2)->lhs();
+                    if (type->variant() == DISTRIBUTE_OP)
+                    {
+                        list = st->expr(0);
+                        while (list)
+                        {
+                            directives["distribute"].push_back(parseDistribution(byPos, list->lhs()->symbol(), type->lhs(), st->lineNumber()));
+                            list = list->rhs();
+                        }
+                    }
+                    else if (type->variant() == ALIGN_OP)
+                    {
+                        list = st->expr(0);
+                        while (list)
+                        {
+                            directives["align"].push_back(parseAlign(byPos, list->lhs()->symbol(), type->rhs()->symbol(), type->lhs(), type->rhs()->lhs(), st->lineNumber()));
+                            list = list->rhs();
+                        }
+                    }
+                }
                     break;
                 case DVM_DISTRIBUTE_DIR:
-                    dirs << "1;" << "DISTRIBUTE;" << st->expr(1)->unparse() << ";" << st->expr(0)->unparse() << ";\n";
-                    break;
+                    directives["distribute"].push_back(parseDistribution(byPos, st->expr(0)->lhs()->symbol(), st->expr(1), st->lineNumber()));
+                    break;                
                 case DVM_ALIGN_DIR:
-                    dirs << "1;" << "ALIGN;" << st->expr(0)->unparse() << "(" << st->expr(1)->unparse() << ");" << st->expr(2)->unparse() << ";\n";
+                    directives["align"].push_back(parseAlign(byPos, st->expr(0)->lhs()->symbol(), st->expr(2)->symbol(), st->expr(1), st->expr(2)->lhs(), st->lineNumber()));
                     break;
                 case DVM_SHADOW_DIR:
-                    dirs << "1;" << "SHADOW;" << st->expr(0)->unparse() << "(" << st->expr(1)->unparse() << ");\n";
+                    //dirs << "1;" << "SHADOW;" << st->expr(0)->unparse() << "(" << st->expr(1)->unparse() << ");\n";
                     break;
                 case DVM_REMOTE_ACCESS_DIR:
                     {
@@ -314,14 +641,14 @@ void parseDvmDirForPredictor(const map<string, vector<FuncInfo*>>& allFuncInfo,
                             printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                         }
 
-                        dirs << info.getExecutedCount() << ";" << "REMOTE_ACCESS;";
+                        //dirs << info.getExecutedCount() << ";" << "REMOTE_ACCESS;";
                         list = st->expr(0);
                         while (list)
                         {
-                            dirs << list->lhs()->unparse() << ";";
+                            //dirs << list->lhs()->unparse() << ";";
                             list = list->rhs();
                         }
-                        dirs << "\n";
+                        //dirs << "\n";
                         break;
                     }
                 default:
@@ -332,5 +659,10 @@ void parseDvmDirForPredictor(const map<string, vector<FuncInfo*>>& allFuncInfo,
         }
     }
 
-    dirs.close();
+    //printf("%s\n", info.dump(2).c_str());
+    
+    ofstream dump("info.json");
+    dump << info.dump(2) << std::endl;
+    dump.flush();
+    dump.close();
 }

sapfor/experts/Sapfor_2017/_src/Predictor/PredictScheme.h

@@ -55,4 +55,4 @@ public:
 void processFileToPredict(SgFile *file, PredictorStats &predictorCounts);
 
 void calculateStatsForPredictor(const std::map<std::string, std::vector<FuncInfo*>>& allFuncInfo, const std::map<std::string, std::map<int, Gcov_info>>& gCovInfo);
-void parseDvmDirForPredictor(const std::map<std::string, std::vector<FuncInfo*>>& allFuncInfo, const std::map<std::string, std::map<int, Gcov_info>>& gCovInfo);
+void parseDvmDirForPredictor(const std::map<std::tuple<int, std::string, std::string>, std::pair<DIST::Array*, DIST::ArrayAccessInfo*>>& declaredArrays, const std::map<std::string, CommonBlock*>& commonBlocks, const std::map<std::string, std::vector<FuncInfo*>>& allFuncInfo, const std::map<std::string, std::map<int, Gcov_info>>& gCovInfo);

sapfor/experts/Sapfor_2017/_src/PrivateArrays/private_arrays_search.cpp

@@ -0,0 +1,195 @@
+#include <map>
+#include <unordered_set>
+#include <vector>
+#include <queue>
+#include <iostream>
+
+#include "private_arrays_search.h"
+
+void print_info(LoopGraph* loop)
+{
+    std::cout << "loopSymbol: " << loop->loopSymbol << std::endl;
+    for (const auto ops : loop->writeOpsForLoop)
+    {
+        std::cout << "Array name: " << ops.first->GetShortName() << std::endl;
+        for (const auto i : ops.second)
+        {
+            i.printInfo();
+        }
+    }
+    if (!loop->children.empty())
+    {
+        for (const auto child : loop->children)
+        {
+            print_info(child);
+        }
+    }
+}
+
+static bool isParentStmt(SgStatement* stmt, SgStatement* parent)
+{
+    for (; stmt; stmt = stmt->controlParent())
+        if (stmt == parent)
+            return true;
+
+    return false;
+}
+
+/*returns head block and loop*/
+std::pair<SAPFOR::BasicBlock*, std::unordered_set<SAPFOR::BasicBlock*>> GetBasicBlocksForLoop(LoopGraph* loop, std::vector<SAPFOR::BasicBlock*> blocks)
+{
+    std::unordered_set<SAPFOR::BasicBlock*> block_loop;
+    SAPFOR::BasicBlock* head_block = nullptr;
+    auto loop_operator = loop->loop->GetOriginal();
+    for (auto& block : blocks)
+    {
+        if (!block || (block->getInstructions().size() == 0))
+            continue;
+        SgStatement* first = block->getInstructions().front()->getInstruction()->getOperator();
+        SgStatement* last = block->getInstructions().back()->getInstruction()->getOperator();
+        if (isParentStmt(first, loop_operator) && isParentStmt(last, loop_operator))
+        {
+            block_loop.insert(block);
+
+            if ((!head_block) && (first == loop_operator) && (last == loop_operator) &&
+                (block->getInstructions().size() == 2) &&
+                (block->getInstructions().back()->getInstruction()->getOperation() == SAPFOR::CFG_OP::JUMP_IF))
+            {
+                head_block = block;
+            }
+
+        }
+    }
+    return { head_block, block_loop };
+}
+
+LoopGraph* FindLoopForIndex(std::string var_name, LoopGraph* loop)
+{
+    if (loop->loopSymbol == var_name)
+        return loop;
+    else if (loop->children.empty()) {
+        return nullptr;
+    }
+    else
+    {
+        for (auto& child : loop->children)
+        {
+            LoopGraph* res = FindLoopForIndex(var_name, child);
+            if (res)
+                return res;
+        }
+    }
+}
+
+void GetDefUseArray(SAPFOR::BasicBlock* block, LoopGraph* loop, ArrayAccessingIndexes& def, ArrayAccessingIndexes& use) {
+    
+    auto instructions = block->getInstructions();
+    for(int i = 0; i < instructions.size(); i++)
+    {
+        auto instruction = instructions[i];
+        if ((instruction->getInstruction()->getOperation() == SAPFOR::CFG_OP::STORE && instruction->getInstruction()->getArg1()->getType() == SAPFOR::CFG_ARG_TYPE::ARRAY) ||
+            (instruction->getInstruction()->getOperation() == SAPFOR::CFG_OP::LOAD && instruction->getInstruction()->getArg2()->getType() == SAPFOR::CFG_ARG_TYPE::ARRAY))
+        {
+            SAPFOR::CFG_OP operation = instruction->getInstruction()->getOperation();
+            std::queue<SAPFOR::Argument*> index_vars;
+            std::string array_name;
+            if(operation == SAPFOR::CFG_OP::STORE)
+                array_name = instruction->getInstruction()->getArg1()->getValue();
+            else
+                array_name = instruction->getInstruction()->getArg2()->getValue();
+            int j = i - 1;
+            while (j >= 0 && instructions[j]->getInstruction()->getOperation() == SAPFOR::CFG_OP::REF)
+            {
+                index_vars.push(instructions[j]->getInstruction()->getArg1());
+            }
+            /*to choose correct dimension*/
+            int n = index_vars.size();
+            if (operation == SAPFOR::CFG_OP::STORE)
+            {
+                if (def[array_name].empty())
+                    def[array_name].resize(n);
+            }
+            else
+            {
+                if (use[array_name].empty())
+                    use[array_name].resize(n);
+            }
+            while (!index_vars.empty())
+            {
+                auto var = index_vars.front();
+                ArrayDimension current_dim;
+                if (var->getType() == SAPFOR::CFG_ARG_TYPE::CONST) {
+                    current_dim = { stol(var->getValue()), 0, 1 };
+                }
+                else
+                {
+                    std::string full_name = var->getValue();
+                    int pos = full_name.find('%');
+                    LoopGraph* current_loop = FindLoopForIndex(full_name.substr(pos), loop);
+                    current_dim = { loop->startVal, loop->stepVal, loop->calculatedCountOfIters };
+                }
+                if (operation == SAPFOR::CFG_OP::STORE)
+                    def[array_name][n - index_vars.size()].push_back(current_dim);
+                else
+                    use[array_name][n - index_vars.size()].push_back(current_dim);
+                index_vars.pop();
+            }
+        }
+    }
+
+}
+
+
+
+
+void FindPrivateArrays(std::map<std::string, std::vector<LoopGraph*>> &loopGraph, std::map<FuncInfo*, std::vector<SAPFOR::BasicBlock*>>& FullIR)
+{   
+    for (const auto& curr_graph_pair: loopGraph)
+    {
+        for (const auto& curr_loop : curr_graph_pair.second)
+        {
+            ArrayAccessingIndexes loopDimensionsInfo;
+            //GetDimensionInfo(curr_loop, loopDimensionsInfo, 0);
+            //print_info(curr_loop);
+        }
+    }
+}
+
+void GetDimensionInfo(LoopGraph* loop, std::map<DIST::Array*, std::vector<std::vector<ArrayDimension>>>& loopDimensionsInfo, int level)
+{
+    std::cout << "line_num: " << loop->lineNum << std::endl;
+    for (const auto& writeOpPairs : loop->writeOpsForLoop)
+    {
+        std::vector<std::vector<ArrayDimension>> arrayDimensions(writeOpPairs.first->GetDimSize());
+        loopDimensionsInfo[writeOpPairs.first] = arrayDimensions;
+        for (const auto& writeOp : writeOpPairs.second)
+        {
+            for (const auto& coeficient_pair : writeOp.coefficients)
+            {
+                int64_t start, step, tripCount;
+                start = loop->startVal * coeficient_pair.first.first + coeficient_pair.first.second;
+                step = loop->stepVal * coeficient_pair.first.first;
+                tripCount = (loop->endVal - coeficient_pair.first.second) / step;
+                if (start <= loop->endVal)
+                {
+                    loopDimensionsInfo[writeOpPairs.first][level].push_back({start, step, tripCount});
+                    std::cout << "level: " << level << std::endl;
+                    std::cout << "start: " << start << std::endl;
+                    std::cout << "step: " << step << std::endl;
+                    std::cout << "trip_count: " << tripCount << std::endl;
+                    std::cout << std::endl;
+                }
+                
+                
+            }
+        }
+    }
+    std::cout << "line_num_after: " << loop->lineNumAfterLoop << std::endl;
+    if (!loop->children.empty())
+    {
+        for (const auto& childLoop : loop->children)
+        {
+            GetDimensionInfo(childLoop, loopDimensionsInfo, level+1);
+        }
+    }
+}

sapfor/experts/Sapfor_2017/_src/PrivateArrays/private_arrays_search.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "../GraphLoop/graph_loops.h"
+#include "../CFGraph/CFGraph.h"
+
+struct ArrayDimension 
+{
+    int64_t start, step, tripCount;
+};
+
+typedef  std::map<std::string, std::vector<std::vector<ArrayDimension>>>  ArrayAccessingIndexes;
+
+void FindPrivateArrays(std::map<std::string, std::vector<LoopGraph*>>& loopGraph, std::map<FuncInfo*, std::vector<SAPFOR::BasicBlock*>>& FullIR);
+void GetDimensionInfo(LoopGraph* loop, std::map<DIST::Array*, std::vector<std::vector<ArrayDimension>>>& loopDimensionsInfo, int level);
+std::set<SAPFOR::BasicBlock> GetBasicBlocksForLoop(LoopGraph* loop, std::vector<SAPFOR::BasicBlock>);

sapfor/experts/Sapfor_2017/_src/Sapfor.cpp

@@ -24,6 +24,8 @@
 #include "ParallelizationRegions/resolve_par_reg_conflicts.h"
 #include "ParallelizationRegions/expand_extract_reg.h"
 
+#include "Transformations/replace_dist_arrays_in_io.h"
+
 #include "Distribution/Distribution.h"
 #include "Distribution/GraphCSR.h"
 #include "Distribution/Arrays.h"
@@ -96,6 +98,8 @@
 
 #include "Inliner/inliner.h"
 
+#include "PrivateArrays/private_arrays_search.h"
+
 #include "dvm.h"
 #include "Sapfor.h"
 #include "Utils/PassManager.h"
@@ -436,7 +440,7 @@ static bool runAnalysis(SgProject &project, const int curr_regime, const bool ne
     int global_err = 0;
 
     map<string, int> same_decls;
-
+    
     for (int i = n - 1; i >= 0; --i)
     {
         createNeededException();
@@ -1567,6 +1571,8 @@ static bool runAnalysis(SgProject &project, const int curr_regime, const bool ne
         if (error)
             internalExit = 1;
     }
+    else if (curr_regime == REMOVE_DIST_ARRAYS_FROM_IO)
+        replaceDistributedArraysInIO(parallelRegions, allFuncInfo, SPF_messages, newCopyDeclToIncl);
     else if (curr_regime == LOOP_GRAPH)
     {
         if (keepFiles)
@@ -1896,9 +1902,11 @@ static bool runAnalysis(SgProject &project, const int curr_regime, const bool ne
     else if (curr_regime == GET_STATS_FOR_PREDICTOR)
     {
         calculateStatsForPredictor(allFuncInfo, gCovInfo);
-        parseDvmDirForPredictor(allFuncInfo, gCovInfo);
+        parseDvmDirForPredictor(declaredArrays, commonBlocks, allFuncInfo, gCovInfo);
+    }
+    else if (curr_regime == FIND_PRIVATE_ARRAYS) {
+        FindPrivateArrays(loopGraph, fullIR);
     }
-
     const float elapsed = duration_cast<milliseconds>(high_resolution_clock::now() - timeForPass).count() / 1000.;
     const float elapsedGlobal = duration_cast<milliseconds>(high_resolution_clock::now() - globalTime).count() / 1000.;
     __spf_print(1, "PROFILE: time for this pass = %f sec (total %f sec)\n", elapsed, elapsedGlobal);
@@ -2333,6 +2341,7 @@ void runPass(const int curr_regime, const char *proj_name, const char *folderNam
         findFunctionsToInclude(true);
         break;
         // all these cases run UNPARSE_FILE after
+    case REMOVE_DIST_ARRAYS_FROM_IO:
     case RENAME_SYMBOLS:
     case RESOLVE_PAR_REGIONS:
     case CREATE_PARALLEL_REGIONS:

sapfor/experts/Sapfor_2017/_src/Sapfor.h

@@ -87,6 +87,8 @@ enum passes {
     REMOVE_DVM_INTERVALS,
     VERIFY_DVM_DIRS,
 
+    REMOVE_DIST_ARRAYS_FROM_IO,
+
     SUBST_EXPR,
     SUBST_EXPR_RD,
     REVERT_SUBST_EXPR,
@@ -180,6 +182,8 @@ enum passes {
     SET_IMPLICIT_NONE,
     RENAME_INLCUDES,
 
+    FIND_PRIVATE_ARRAYS,
+
     TEST_PASS,
     EMPTY_PASS
 };
@@ -260,6 +264,7 @@ static void setPassValues()
     passNames[INSERT_INCLUDES] = "INSERT_INCLUDES";
     passNames[REMOVE_DVM_DIRS] = "REMOVE_DVM_DIRS";
     passNames[VERIFY_DVM_DIRS] = "VERIFY_DVM_DIRS";
+    passNames[REMOVE_DIST_ARRAYS_FROM_IO] = "REMOVE_DIST_ARRAYS_FROM_IO";
     passNames[SUBST_EXPR] = "SUBST_EXPR";
     passNames[SUBST_EXPR_RD] = "SUBST_EXPR_RD";
     passNames[CALL_GRAPH2] = "CALL_GRAPH2";
@@ -363,6 +368,8 @@ static void setPassValues()
     passNames[RENAME_INLCUDES] = "RENAME_INLCUDES";
     passNames[INSERT_NO_DISTR_FLAGS_FROM_GUI] = "INSERT_NO_DISTR_FLAGS_FROM_GUI";
 
+    passNames[FIND_PRIVATE_ARRAYS] = "FIND_PRIVATE_ARRAYS";
+
     passNames[TEST_PASS] = "TEST_PASS";
 }
 

sapfor/experts/Sapfor_2017/_src/Transformations/function_purifying.cpp

@@ -489,6 +489,9 @@ static SgSymbol* getParameter(SgStatement* stat, int n)
 
 static void intentInsert(const FuncInfo* func, SgStatement* headerSt)
 {
+    if (func == NULL)
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
     if (func->funcPointer->variant() == ENTRY_STAT)
         return;
 

sapfor/experts/Sapfor_2017/_src/Transformations/replace_dist_arrays_in_io.cpp

@@ -0,0 +1,478 @@
+#include "replace_dist_arrays_in_io.h"
+
+#include "../ParallelizationRegions/resolve_par_reg_conflicts.h"
+
+#include <string>
+#include <map>
+#include <set>
+
+using std::map;
+using std::set;
+using std::string;
+using std::vector;
+using std::to_string;
+using std::make_pair;
+
+#define DEBUG_TRACE 0
+
+static void findArrays(SgExpression* exp, set<SgSymbol*>& arrays)
+{
+    if (exp)
+    {
+        if (isArrayRef(exp))
+            arrays.insert(exp->symbol());
+
+        findArrays(exp->lhs(), arrays);
+        findArrays(exp->rhs(), arrays);
+    }
+}
+
+static void populateDistributedIoArrays(map<DIST::Array*, set<SgStatement*>>& arrays, SgStatement* stat)
+{
+    auto var = stat->variant();
+
+    if (var != READ_STAT && var != PRINT_STAT && var != WRITE_STAT)
+        return;
+
+    // check if such IO allowed in dvm: 
+    // list should consist only of single array and format string should be *
+
+    bool need_replace = false;
+
+    SgExpression* ioList = stat->expr(0);
+
+    if (!ioList)
+        return;
+
+    if (ioList->variant() != EXPR_LIST)
+        return;
+
+    if (ioList->rhs() == NULL)
+    {
+        SgExpression* arg = ioList->lhs();
+        if (!arg)
+            return;
+
+        if (!isArrayRef(arg))
+            return;
+
+        if (arg->lhs())
+            need_replace = true;
+    }
+    else
+    {
+        need_replace = true;
+    }
+
+    if (!need_replace)
+    {
+        switch (var)
+        {
+        case PRINT_STAT:
+        {
+            SgExpression* fmt = stat->expr(1);
+            if (!fmt || fmt->variant() != SPEC_PAIR || fmt->lhs()->variant() != KEYWORD_VAL)
+            {
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                return;
+            }
+
+            if (fmt->rhs()->variant() != KEYWORD_VAL || fmt->rhs()->sunparse() != "*")
+                need_replace = true;
+
+
+            break;
+        }
+        case READ_STAT:
+        case WRITE_STAT:
+        {
+
+            SgExpression* spec = stat->expr(1);
+            __spf_print(DEBUG_TRACE, "[%d: %s (%d)]\n", 2000, spec->rhs()->unparse(), spec->rhs()->variant());
+            if (!spec || spec->variant() != EXPR_LIST ||
+                spec->lhs()->variant() != SPEC_PAIR ||
+                !spec->rhs() || !spec->rhs()->lhs() || spec->rhs()->lhs()->variant() != SPEC_PAIR)
+            {
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                return;
+            }
+
+            SgExpression* unit_val = spec->lhs()->rhs(), * fmt_val = spec->rhs()->lhs()->rhs();
+
+            if (unit_val->variant() != KEYWORD_VAL || unit_val->sunparse() != "*" ||
+                fmt_val->variant() != KEYWORD_VAL || fmt_val->sunparse() != "*")
+                need_replace = true;
+
+            break;
+        }
+        default:
+            break;
+        }
+    }
+
+    if (!need_replace)
+        return;
+
+    set<SgSymbol*> found_arrays;
+
+    for (int i = 0; i < 3; i++)
+        findArrays(stat->expr(i), found_arrays);
+
+    for (auto* by_symb : found_arrays)
+    {
+        string array_name = string(by_symb->identifier());
+        DIST::Array* array_p = getArrayFromDeclarated(declaratedInStmt(by_symb), array_name);
+        if (array_p && array_p->GetDistributeFlagVal() == Distribution::distFlag::IO_PRIV && arrays[array_p].insert(stat).second)
+            __spf_print(DEBUG_TRACE, "[%d]: add array %s\n", stat->lineNumber(), array_p->GetName().c_str());
+    }
+
+    __spf_print(DEBUG_TRACE, "[replace]\n");
+}
+
+static void replaceArrayRec(SgSymbol* arr, SgSymbol* replace_by, SgExpression* exp, bool& has_read, bool& has_write, bool from_read, bool from_write)
+{
+    if (!exp)
+        return;
+
+    if (exp->symbol() && strcmp(exp->symbol()->identifier(), arr->identifier()) == 0)
+    {
+        has_read |= from_read;
+        has_write |= from_write;
+        exp->setSymbol(replace_by);
+    }
+
+    switch (exp->variant())
+    {
+        case FUNC_CALL:
+        {
+            replaceArrayRec(arr, replace_by, exp->rhs(), has_read, has_write, true, false);
+            replaceArrayRec(arr, replace_by, exp->lhs(), has_read, has_write, true, true);
+            break;
+        }
+        case EXPR_LIST:
+        {
+            replaceArrayRec(arr, replace_by, exp->lhs(), has_read, has_write, from_read, from_write);
+            replaceArrayRec(arr, replace_by, exp->rhs(), has_read, has_write, from_read, from_write);
+            break;
+        }
+        default:
+        {
+            replaceArrayRec(arr, replace_by, exp->lhs(), has_read, has_write, true, false);
+            replaceArrayRec(arr, replace_by, exp->rhs(), has_read, has_write, true, false);
+            break;
+        }
+    }
+}
+
+static void replaceArrayRec(SgSymbol* arr, SgSymbol* replace_by, SgStatement* st, bool& has_read, bool& has_write)
+{
+    if (!st)
+        return;
+
+    switch (st->variant())
+    {
+        case ASSIGN_STAT:
+        case READ_STAT:
+        {
+            replaceArrayRec(arr, replace_by, st->expr(0), has_read, has_write, false, true);
+            replaceArrayRec(arr, replace_by, st->expr(1), has_read, has_write, true, false);
+            break;
+        }
+        case PROC_STAT:
+        case FUNC_STAT:
+        {
+            replaceArrayRec(arr, replace_by, st->expr(0), has_read, has_write, true, false);
+            replaceArrayRec(arr, replace_by, st->expr(1), has_read, has_write, true, true);
+            break;
+        }
+        default:
+        {
+            for (int i = 0; i < 3; i++)
+                replaceArrayRec(arr, replace_by, st->expr(i), has_read, has_write, true, false);
+
+            break;
+        }
+    }
+}
+
+static void copyArrayBetweenStatements(SgSymbol* replace_symb, SgSymbol* replace_by, SgStatement* start, SgStatement* last)
+{
+    while (start->lexNext() && !isSgExecutableStatement(start->lexNext()))
+        start = start->lexNext();
+
+    auto* stop = last->lexNext();
+
+    bool has_read = false, has_write = false;
+
+    for (auto* st = start; st != stop; st = st->lexNext())
+        replaceArrayRec(replace_symb, replace_by, st, has_read, has_write);
+
+
+    if (has_read)
+    {
+        // A_copy = A
+        SgAssignStmt* assign = new SgAssignStmt(*new SgArrayRefExp(*replace_by), *new SgArrayRefExp(*replace_symb));
+        assign->setlineNumber(getNextNegativeLineNumber()); // before region
+        auto* parent = start->controlParent();
+        if (parent && parent->lastNodeOfStmt() == start)
+            parent = parent->controlParent();
+
+        start->insertStmtAfter(*assign, *parent);
+    }
+
+    if (has_write)
+    {
+        // A = A_reg
+        SgAssignStmt* assign = new SgAssignStmt(*new SgArrayRefExp(*replace_symb), *new SgArrayRefExp(*replace_by));
+        //TODO: bug with insertion
+        //assign->setlineNumber(getNextNegativeLineNumber()); // after region
+        last->insertStmtBefore(*assign, *(last->controlParent()));
+    }
+}
+
+static void replaceArrayInFragment(DIST::Array* arr, const set<SgStatement*> usages, SgSymbol* replace_by, SgStatement* start, SgStatement* last, const string& filename)
+{
+    while (start->lexNext() && !isSgExecutableStatement(start->lexNext()))
+        start = start->lexNext();
+
+    auto* replace_symb = arr->GetDeclSymbol();
+
+    set<SgStatement*> not_opened, not_closed, copied;
+
+    for (auto* it = start; it; it = it->controlParent())
+        not_opened.insert(it);
+
+    for (auto* it = last; it; it = it->controlParent())
+        not_closed.insert(it);
+
+    for (auto* io_stmt : usages)
+    {
+        bool already_copied = false;
+        SgStatement* copy_scope = NULL;
+
+        for (auto* par = io_stmt; par; par = par->controlParent())
+        {
+            if (copied.find(par) != copied.end())
+            {
+                already_copied = true;
+                break;
+            }
+            else if (not_opened.find(par) != not_opened.end() || not_closed.find(par) != not_closed.end())
+            {
+                copy_scope = par;
+                break;
+            }
+        }
+
+
+        if (already_copied)
+            continue;
+
+        auto* scope_start = copy_scope, * scope_end = copy_scope->lastNodeOfStmt();
+        __spf_print(DEBUG_TRACE, "[scope to copy] %d\n", copy_scope->lineNumber());
+
+        if (not_opened.find(copy_scope) != not_opened.end())
+        {
+            auto* from = start->lastNodeOfStmt() ? start->lastNodeOfStmt() : start;
+            for (auto* st = from; st; st = st->controlParent())
+            {
+                __spf_print(DEBUG_TRACE, "[find start of parent %d] %d\n", copy_scope->lineNumber(), st->lineNumber());
+
+                if (st->controlParent() == copy_scope)
+                {
+                    scope_start = st->lastNodeOfStmt() ? st->lastNodeOfStmt() : st;
+                    break;
+                }
+            }
+        }
+
+        if (not_closed.find(copy_scope) != not_closed.end())
+        {
+            for (auto* st = last; st; st = st->controlParent())
+            {
+                __spf_print(DEBUG_TRACE, "[find end of parent %d] %d\n", copy_scope->lineNumber(), st->lineNumber());
+
+                if (st->controlParent() == copy_scope)
+                {
+                    scope_end = st;
+                    break;
+                }
+            }
+        }
+
+        copyArrayBetweenStatements(replace_symb, replace_by, scope_start, scope_end);
+        __spf_print(DEBUG_TRACE, "[copy %s] [%d, %d]\n", arr->GetName().c_str(), scope_start->lineNumber(), scope_end->lineNumber());
+        copied.insert(copy_scope);
+    }
+}
+
+static bool ioReginBound(SgStatement* stat, SgStatement* last_io_bound)
+{
+    auto var = stat->variant();
+
+    if (var == PROC_STAT || var == FUNC_STAT || var == PROG_HEDR || var == FUNC_HEDR || var == PROC_HEDR || var == FUNC_STAT || var == FOR_NODE || var == LOOP_NODE)
+        return true;
+
+    if (last_io_bound && last_io_bound->lastNodeOfStmt() && last_io_bound->lastNodeOfStmt() == stat)
+        return true;
+
+    int parent_var;
+
+    if (var == CONTROL_END &&
+        ((parent_var = stat->controlParent()->variant()) == PROG_HEDR ||
+            parent_var == PROC_HEDR || parent_var == FUNC_HEDR))
+    {
+        return true;
+    }
+
+    return false;
+}
+
+void replaceDistributedArraysInIO(vector<ParallelRegion*>& regions,
+                                  const map<string, vector<FuncInfo*>>& allFuncInfo,
+                                  map<string, vector<Messages>>& SPF_messages,
+                                  map<string, map<int, set<string>>>& newDeclsToInclude)
+{
+    map<DIST::Array*, SgSymbol*> created_copies;
+    map<string, map<int, set<string>>> copied;
+
+    for (auto& region : regions)
+    {
+        __spf_print(DEBUG_TRACE, "[%s]: enter region\n", region->GetName().c_str());
+
+        for (auto& linesByFile : region->GetAllLinesToModify())
+        {
+            const auto& filename = linesByFile.first;
+
+            if (SgFile::switchToFile(filename) < 0) {
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                return;
+            }
+
+            for (auto& lines : linesByFile.second) {
+                __spf_print(DEBUG_TRACE, "[fragment] %s: %d:%d %d\n", filename.c_str(), lines.lines.first,
+                    lines.lines.second, lines.isImplicit());
+
+                SgStatement* curr_stmt, * end;
+
+                if (lines.isImplicit())
+                {
+
+                    curr_stmt = current_file->SgStatementAtLine(lines.lines.first);
+                    end = current_file->SgStatementAtLine(lines.lines.second);
+
+                    if (end)
+                        end = end->lexNext();
+                }
+                else
+                {
+                    curr_stmt = lines.stats.first->GetOriginal();
+                    end = lines.stats.second->GetOriginal()->lexNext();
+                }
+
+                map<DIST::Array*, set<SgStatement*>> need_replace;
+                SgStatement* last_io_bound = NULL;
+
+
+                while (curr_stmt != end)
+                {
+                    if (!curr_stmt)
+                        break;
+
+                    auto var = curr_stmt->variant();
+
+
+                    if (var == PROC_HEDR || var == PROG_HEDR || var == FUNC_HEDR)
+                    {
+                        curr_stmt = curr_stmt->lexNext();
+                        while (curr_stmt && !isSgExecutableStatement(curr_stmt))
+                        {
+                            last_io_bound = curr_stmt;
+                            curr_stmt = curr_stmt->lexNext();
+                        }
+
+                        if (!curr_stmt)
+                            break;
+                    }
+
+                    if (ioReginBound(curr_stmt, last_io_bound))
+                    {
+                        if (last_io_bound)
+                        {
+                            __spf_print(DEBUG_TRACE, "[io region] [%d, %d]\n", last_io_bound->lineNumber(), curr_stmt->lineNumber());
+
+                            for (const auto& p : need_replace)
+                            {
+                                auto it = created_copies.find(p.first);
+                                if (it != created_copies.end())
+                                    replaceArrayInFragment(p.first, p.second, it->second, last_io_bound, curr_stmt, filename);
+                                else
+                                {
+                                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                                    return;
+                                }
+                            }
+                        }
+
+                        need_replace.clear();
+                        last_io_bound = curr_stmt;
+                    }
+
+                    __spf_print(DEBUG_TRACE, "[line] %d (%d)\n", curr_stmt->lineNumber(), curr_stmt->variant());
+
+                    for (int i = 0; i < 3; i++)
+                    {
+                        if (curr_stmt->expr(i))
+                        {
+                            __spf_print(DEBUG_TRACE, "[%d: %s (%d)]\n", i, curr_stmt->expr(i)->unparse(), curr_stmt->expr(i)->variant());
+                        }
+                    }
+
+                    populateDistributedIoArrays(need_replace, curr_stmt);
+
+                    for (const auto& by_array_to_copy : need_replace)
+                    {
+                        auto* array_to_copy = by_array_to_copy.first;
+                        auto it = created_copies.find(array_to_copy);
+
+                        if (it == created_copies.end())
+                        {
+                            bool fromModule = (array_to_copy->GetLocation().first == DIST::l_MODULE);
+                            const string locationName = array_to_copy->GetLocation().second;
+
+                            auto place = *array_to_copy->GetDeclInfo().begin();
+                            string fileName = place.first;
+                            string suffix = "_io_l";
+
+                            if (fromModule)
+                                suffix = "_io_m";
+
+                            auto origCopy = copyArray(place, array_to_copy, linesByFile.second, suffix + to_string(region->GetId()), fileName, newDeclsToInclude, copied);
+                            SgStatement* decl = SgStatement::getStatementByFileAndLine(place.first, place.second);
+
+                            if(decl)
+                                decl = decl->lexNext();
+
+                            if(decl)
+                            {
+                                string dir_str;
+                                if (decl->comments())
+                                {
+                                    string str_comment = string(decl->comments());
+                                    if(str_comment.size() && str_comment.back() != '\n')
+                                        dir_str += "\n";
+                                }
+
+                                dir_str += "!$SPF ANALYSIS(PROCESS_PRIVATE(" + string(origCopy.second->identifier()) + "))\n";
+                                decl->addComment(dir_str.c_str());
+                            }
+                            created_copies.insert({ array_to_copy, origCopy.second });
+                        }
+                    }
+
+                    curr_stmt = curr_stmt->lexNext();
+                }
+            }
+        }
+    }
+}

sapfor/experts/Sapfor_2017/_src/Transformations/replace_dist_arrays_in_io.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "../ParallelizationRegions/ParRegions.h"
+#include "../Utils/SgUtils.h"
+#include "../Utils/errors.h"
+#include "../GraphCall/graph_calls.h"
+
+void replaceDistributedArraysInIO(std::vector<ParallelRegion*>& regions, 
+                                  const std::map<std::string, std::vector<FuncInfo*>>& allFuncInfo, 
+                                  std::map<std::string, std::vector<Messages>>& SPF_messages,
+                                  std::map<std::string, std::map<int, std::set<std::string>>>& newDeclsToInclude);

sapfor/experts/Sapfor_2017/_src/Utils/PassManager.h

@@ -257,6 +257,8 @@ void InitPassesDependencies(map<passes, vector<passes>> &passDepsIn, set<passes>
 
     list({ REVERT_SUBST_EXPR_RD, CONVERT_LOOP_TO_ASSIGN }) <= Pass(RESOLVE_PAR_REGIONS);
 
+    list({ REVERT_SUBST_EXPR_RD, CONVERT_LOOP_TO_ASSIGN, FILL_PAR_REGIONS}) <= Pass(REMOVE_DIST_ARRAYS_FROM_IO);
+
     Pass(REVERT_SUBST_EXPR_RD) <= Pass(EXPAND_EXTRACT_PAR_REGION);
     
     Pass(FILL_PAR_REGIONS) <= Pass(PRINT_PAR_REGIONS_ERRORS);
@@ -312,6 +314,8 @@ void InitPassesDependencies(map<passes, vector<passes>> &passDepsIn, set<passes>
 
     list({ VERIFY_INCLUDES, CORRECT_VAR_DECL }) <= Pass(SET_IMPLICIT_NONE);
 
+    list({ CALL_GRAPH2, LOOP_ANALYZER_DATA_DIST_S1 }) <= Pass(FIND_PRIVATE_ARRAYS);
+
     passesIgnoreStateDone.insert({ CREATE_PARALLEL_DIRS, INSERT_PARALLEL_DIRS, INSERT_SHADOW_DIRS, EXTRACT_PARALLEL_DIRS, 
                                    EXTRACT_SHADOW_DIRS, CREATE_REMOTES, UNPARSE_FILE, REMOVE_AND_CALC_SHADOW,
                                    REVERSE_CREATED_NESTED_LOOPS, PREDICT_SCHEME, CALCULATE_STATS_SCHEME, REVERT_SPF_DIRS, CLEAR_SPF_DIRS, TRANSFORM_SHADOW_IF_FULL,

sapfor/experts/Sapfor_2017/_src/Utils/SgUtils.cpp

@@ -2726,7 +2726,8 @@ SgStatement* makeDeclaration(SgStatement* curr, const vector<SgSymbol*>& sIn, ve
         decl->setFileName(place->fileName());
         decl->setFileId(place->getFileId());
         decl->setProject(place->getProject());
-        decl->setlineNumber(place->lineNumber());
+        decl->setlineNumber(getNextNegativeLineNumber());
+        //decl->setlineNumber(place->lineNumber());
 
         place->insertStmtBefore(*decl, *scope);
     }

sapfor/experts/Sapfor_2017/_src/Utils/version.h

@@ -1,3 +1,3 @@
 #pragma once
 
-#define VERSION_SPF "2366"
+#define VERSION_SPF "2372"

sapfor/experts/Sapfor_2017/_src/VisualizerCalls/get_information.cpp

@@ -1732,6 +1732,14 @@ int SPF_ResolveParallelRegionConflicts(void*& context, int winHandler, short *op
     return simpleTransformPass(RESOLVE_PAR_REGIONS, options, projName, folderName, output, outputSize, outputMessage, outputMessageSize);
 }
 
+int SPF_RemoveDistArraysFromIO(void*& context, int winHandler, short* options, short* projName, short* folderName, short*& output,
+                               int*& outputSize, short*& outputMessage, int*& outputMessageSize)
+{
+    MessageManager::clearCache();
+    MessageManager::setWinHandler(winHandler);
+    return simpleTransformPass(REMOVE_DIST_ARRAYS_FROM_IO, options, projName, folderName, output, outputSize, outputMessage, outputMessageSize);
+
+}
 int SPF_PrivateExpansion(void*& context, int winHandler, short *options, short *projName, short *folderName, short *&output,
                          int *&outputSize, short *&outputMessage, int *&outputMessageSize)
 {
@@ -2583,6 +2591,8 @@ const wstring Sapfor_RunTransformation(const char* transformName_c, const char*
         retCode = SPF_InsertIncludesPass(context, winHandler, optSh, projSh, fold, (char*)addOpt_c, output, outputSize, outputMessage, outputMessageSize);
     else if (whichRun == "SPF_ResolveParallelRegionConflicts")
         retCode = SPF_ResolveParallelRegionConflicts(context, winHandler, optSh, projSh, fold, output, outputSize, outputMessage, outputMessageSize);
+    else if (whichRun == "SPF_RemoveDistArraysFromIO")
+        retCode = SPF_RemoveDistArraysFromIO(context, winHandler, optSh, projSh, fold, output, outputSize, outputMessage, outputMessageSize);
     else if (whichRun == "SPF_LoopEndDoConverterPass")
         retCode = SPF_LoopEndDoConverterPass(context, winHandler, optSh, projSh, fold, output, outputSize, outputMessage, outputMessageSize);
     else if (whichRun == "SPF_CreateParallelVariant")