M/VisualSapfor
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Доделал Planner. Для компиляции тоже заменил старый Do на DoWithSchedule

Browse Source 
Добавил сохранение в архив задач на компиляцию.
Закомментирова info txt который почему то в архив клался пустым, вместо него кладу в архив файлы TaskState и total_time.
Адаптировал анализ под распаковку архива.
При сохранении в бд, удаляю задачи на компиляцию.
This commit is contained in:
02090095
2023-12-04 01:43:08 +03:00
parent 39ee78da3d
commit 1f8ebdc9a2
9 changed files with 48 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
.idea/workspace.xml generated
View File

@@ -8,20 +8,14 @@
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="e42177c3-2328-4b27-8a01-35779b2beb99" name="Default Changelist" comment=""> <list default="true" id="e42177c3-2328-4b27-8a01-35779b2beb99" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/Array.h" beforeDir="false" /> <change beforePath="$PROJECT_DIR$/src/GlobalData/Tasks/TaskState.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/GlobalData/Tasks/TaskState.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/CompilationSupervisor.h" beforeDir="false" /> <change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/TestingServer.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestingServer.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/CompilationTask.h" beforeDir="false" /> <change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/TestsSupervisor_2022.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/TestsSupervisor_2022.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/File.h" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/Global.h" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/Planner.cpp" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/RunSupervisor.h" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/RunTask.h" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/String.h" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/Supervisor.h" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/Task.h" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/Text.h" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/Planner/Utils.h" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/All/RemoteInitialiseUser.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/All/RemoteInitialiseUser.java" afterDir="false" /> <change beforePath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/All/RemoteInitialiseUser.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/All/RemoteInitialiseUser.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/files/Planner/Planner.cpp" beforeDir="false" afterPath="$PROJECT_DIR$/src/files/Planner/Planner.cpp" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/files/Planner/RunTask.h" beforeDir="false" afterPath="$PROJECT_DIR$/src/files/Planner/RunTask.h" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/files/Planner/Supervisor.h" beforeDir="false" afterPath="$PROJECT_DIR$/src/files/Planner/Supervisor.h" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/files/Planner/Task.h" beforeDir="false" afterPath="$PROJECT_DIR$/src/files/Planner/Task.h" afterDir="false" />
</list> </list>
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" /> <option name="HIGHLIGHT_CONFLICTS" value="true" />

2
src/GlobalData/Tasks/TaskState.java
View File

@@ -67,7 +67,7 @@ public enum TaskState implements StatusEnum {
case Canceled: case Canceled:
return "отменена"; return "отменена";
case InternalError: case InternalError:
return "внутренняя ошибка"; return "отсутствует файл состояния";
case WrongTestFormat: case WrongTestFormat:
return "неверный формат"; return "неверный формат";
case FailedToQueue: case FailedToQueue:

6
src/TestingSystem/Common/TestingServer.java
View File

@@ -402,11 +402,11 @@ public class TestingServer extends RepositoryServer<TestsDatabase> {
public void UpdateTestTasks() throws Exception { public void UpdateTestTasks() throws Exception {
Vector<TestCompilationTask> compilationTasks = (Vector<TestCompilationTask>) request.object; Vector<TestCompilationTask> compilationTasks = (Vector<TestCompilationTask>) request.object;
account_db.BeginTransaction(); account_db.BeginTransaction();
//сами задачи на компиляцию не сохраняем. незачем.
for (TestCompilationTask compilationTask : compilationTasks) { for (TestCompilationTask compilationTask : compilationTasks) {
compilationTask.ChangeDate = new Date().getTime(); account_db.Delete(compilationTask);
account_db.Update(compilationTask);
for (TestRunTask runTask : compilationTask.runTasks) { for (TestRunTask runTask : compilationTask.runTasks) {
compilationTask.ChangeDate = new Date().getTime(); runTask.ChangeDate = new Date().getTime();
account_db.Update(runTask); account_db.Update(runTask);
} }
} }

104
src/TestingSystem/DVM/TestsSupervisor_2022.java
View File

@@ -12,6 +12,7 @@ import TestingSystem.DVM.Tasks.TestRunTask;
import TestingSystem.DVM.Tasks.TestTask; import TestingSystem.DVM.Tasks.TestTask;
import TestingSystem.DVM.TasksPackage.TasksPackage; import TestingSystem.DVM.TasksPackage.TasksPackage;
import TestingSystem.DVM.TasksPackage.TasksPackageState; import TestingSystem.DVM.TasksPackage.TasksPackageState;
import Visual_DVM_2021.Passes.All.UnzipFolderPass;
import com.jcraft.jsch.ChannelSftp; import com.jcraft.jsch.ChannelSftp;
import javafx.util.Pair; import javafx.util.Pair;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
@@ -37,7 +38,6 @@ public class TestsSupervisor_2022 {
connection = connection_in; connection = connection_in;
tasksPackage = tasksPackage_in; tasksPackage = tasksPackage_in;
compilationTasks = tasks_in; compilationTasks = tasks_in;
planner.Print(getClass().getSimpleName() + ": найдено задач на компиляцию: " + compilationTasks.size());
packageRemoteWorkspace = new RemoteFile(tasksPackage.user_workspace + "/tests", String.valueOf(tasksPackage.id), true); packageRemoteWorkspace = new RemoteFile(tasksPackage.user_workspace + "/tests", String.valueOf(tasksPackage.id), true);
packageLocalWorkspace = new File(Global.PackagesDirectory, String.valueOf(tasksPackage.id)); packageLocalWorkspace = new File(Global.PackagesDirectory, String.valueOf(tasksPackage.id));
} }
@@ -170,53 +170,12 @@ public class TestsSupervisor_2022 {
} }
public void DownloadResults() throws Exception { public void DownloadResults() throws Exception {
Utils.CheckDirectory(packageLocalWorkspace); Utils.CheckDirectory(packageLocalWorkspace);
for (TestCompilationTask testCompilationTask : compilationTasks) { RemoteFile remote_results_archive = new RemoteFile(packageRemoteWorkspace.full_name, "results.zip");
//------------>>> File results_archive = new File(packageLocalWorkspace, "results.zip");
if (TryDownloadTask(testCompilationTask)) { if (connection.Exists(packageRemoteWorkspace.full_name, "results.zip")) {
for (TestRunTask testRunTask : testCompilationTask.runTasks) { connection.getSingleFile(remote_results_archive, results_archive, 0);
TryDownloadTask(testRunTask); UnzipFolderPass unzipFolderPass = new UnzipFolderPass();
} unzipFolderPass.Do(results_archive.getAbsolutePath(), packageLocalWorkspace.getAbsolutePath(), false);
} else {
//задача на компиляцию не состоялась. значит и все ее задачи на запуск тоже.
for (TestRunTask testRunTask : testCompilationTask.runTasks) {
testRunTask.state = TaskState.Canceled;
planner.UpdateTask(testRunTask);
}
}
//--->>>>>>>>>
}
}
public boolean TryDownloadTask(TestTask testTask) throws Exception {
if (
!testTask.state.equals(TaskState.ResultsDownloaded) &&
!testTask.state.equals(TaskState.Canceled)
) {
File taskLocalWorkspace = Paths.get(packageLocalWorkspace.getAbsolutePath(), String.valueOf(testTask.id)).toFile();
RemoteFile taskRemoteWorkspace = new RemoteFile(packageRemoteWorkspace.full_name, String.valueOf(testTask.id));
Utils.CheckDirectory(taskLocalWorkspace);
if (connection.Exists(packageRemoteWorkspace.full_name, String.valueOf(testTask.id))) {
CheckTaskFile(taskRemoteWorkspace, taskLocalWorkspace, Constants.out_file);
CheckTaskFile(taskRemoteWorkspace, taskLocalWorkspace, Constants.err_file);
CheckTaskFile(taskRemoteWorkspace, taskLocalWorkspace, Constants.time_file);
CheckTaskFile(taskRemoteWorkspace, taskLocalWorkspace, "TaskState");
if (testTask instanceof TestRunTask)
CheckTaskFile(taskRemoteWorkspace, taskLocalWorkspace, "sts.gz+");
testTask.state = TaskState.ResultsDownloaded;
planner.UpdateTask(testTask);
return true;
} else {
testTask.state = TaskState.Canceled;
planner.UpdateTask(testTask);
return false;
//нет раб пространства. значит задача не выполнялась.
}
} else return true;
}
public void CheckTaskFile(RemoteFile taskRemoteWorkspace, File taskLocalWorkspace, String fileName) throws Exception {
RemoteFile rFile = new RemoteFile(taskRemoteWorkspace.full_name, fileName);
File lFile = Paths.get(taskLocalWorkspace.getAbsolutePath(), fileName).toFile();
if (connection.Exists(taskRemoteWorkspace.full_name, fileName)) {
connection.getSingleFile(rFile, lFile, 0);
} }
} }
public void AnalyseResults() throws Exception { public void AnalyseResults() throws Exception {
@@ -226,7 +185,6 @@ public class TestsSupervisor_2022 {
for (TestCompilationTask testCompilationTask : compilationTasks) { for (TestCompilationTask testCompilationTask : compilationTasks) {
ct_count++; ct_count++;
if (CheckTask(testCompilationTask)) { if (CheckTask(testCompilationTask)) {
// planner.UpdateTask(testCompilationTask);
for (TestRunTask testRunTask : testCompilationTask.runTasks) { for (TestRunTask testRunTask : testCompilationTask.runTasks) {
rt_count++; rt_count++;
testRunTask.compilation_state = testCompilationTask.state; testRunTask.compilation_state = testCompilationTask.state;
@@ -237,7 +195,6 @@ public class TestsSupervisor_2022 {
} else { } else {
CheckTask(testRunTask); CheckTask(testRunTask);
} }
// planner.UpdateTask(testRunTask);
if (testRunTask.state.equals(TaskState.Finished)) { if (testRunTask.state.equals(TaskState.Finished)) {
//анализ задачи на запуск. //анализ задачи на запуск.
List<String> output_lines = Arrays.asList(testRunTask.output.split("\n")); List<String> output_lines = Arrays.asList(testRunTask.output.split("\n"));
@@ -261,55 +218,33 @@ public class TestsSupervisor_2022 {
testRunTask.progress = results.getValue(); testRunTask.progress = results.getValue();
testRunTask.CleanTime = Utils.parseCleanTime(testRunTask.output); testRunTask.CleanTime = Utils.parseCleanTime(testRunTask.output);
} }
File local_sts_text = Utils.getTempFileName("sts_text");
Vector<ChannelSftp.LsEntry> files = connection.sftpChannel.ls(testRunTask.remote_workspace);
for (ChannelSftp.LsEntry file : files) {
if (file.getFilename().equals("sts.gz+")) {
RemoteFile remote_sts = new RemoteFile(
testRunTask.remote_workspace, file.getFilename(), false);
RemoteFile remote_sts_text = new RemoteFile(
testRunTask.remote_workspace, "statistic.txt", false);
try {
connection.ShellCommand(Utils.DQuotes(tasksPackage.dvm_drv) + " pa " +
Utils.DQuotes(remote_sts.full_name) + " " + Utils.DQuotes(remote_sts_text.full_name));
connection.getSingleFile(remote_sts_text, local_sts_text, 10240);
} catch (Exception ex) {
ex.printStackTrace();
} }
if (local_sts_text.exists()) {
try {
testRunTask.statistic = FileUtils.readFileToString(local_sts_text, Charset.defaultCharset());
} catch (Exception e) {
e.printStackTrace();
}
}
break;
}
}
}
//planner.UpdateTask(testRunTask);
} }
} }
} }
System.out.println("ct_count=" + ct_count + " rt count=" + rt_count); System.out.println("ct_count=" + ct_count + " rt count=" + rt_count);
//теперь обновить их единым списком. //теперь обновить их единым списком, и удалить задачи на компиляцию.
planner.ServerCommand(ServerCode.UpdateTestTasks, compilationTasks); planner.ServerCommand(ServerCode.UpdateTestTasks, compilationTasks);
} }
public boolean CheckTask(TestTask testTask) throws Exception { public boolean CheckTask(TestTask testTask) throws Exception {
if (testTask.state.equals(TaskState.ResultsDownloaded)) { File taskWorkspace = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", String.valueOf(testTask.id)).toFile();
File taskWorkspace = Paths.get(packageLocalWorkspace.getAbsolutePath(), String.valueOf(testTask.id)).toFile(); if (taskWorkspace.exists()) {
System.out.println("id=" + testTask.id + ": path=" + taskWorkspace.getAbsolutePath()); System.out.println("id=" + testTask.id + ": path=" + taskWorkspace.getAbsolutePath());
File stateFile = Paths.get(taskWorkspace.getAbsolutePath(), "TaskState").toFile(); File stateFile = new File(taskWorkspace, "TaskState");
File outFile = Paths.get(taskWorkspace.getAbsolutePath(), Constants.out_file).toFile(); File outFile = new File(taskWorkspace, Constants.out_file);
File errFile = Paths.get(taskWorkspace.getAbsolutePath(), Constants.err_file).toFile(); File errFile = new File(taskWorkspace.getAbsolutePath(), Constants.err_file);
File timeFile = Paths.get(taskWorkspace.getAbsolutePath(), Constants.time_file).toFile(); File timeFile = new File(taskWorkspace.getAbsolutePath(), Constants.time_file);
File stsFile = new File(taskWorkspace.getAbsolutePath(), "statistic.txt");
if (outFile.exists()) if (outFile.exists())
testTask.output = FileUtils.readFileToString(outFile); testTask.output = FileUtils.readFileToString(outFile);
if (errFile.exists()) if (errFile.exists())
testTask.errors = FileUtils.readFileToString(errFile); testTask.errors = FileUtils.readFileToString(errFile);
if (timeFile.exists()) if (timeFile.exists())
testTask.Time = Double.parseDouble(Utils.ReadAllText(timeFile)); testTask.Time = Double.parseDouble(Utils.ReadAllText(timeFile));
if ((testTask instanceof TestRunTask) && stsFile.exists()) {
TestRunTask testRunTask = (TestRunTask) testTask;
testRunTask.statistic = FileUtils.readFileToString(stsFile);
}
if (stateFile.exists()) { if (stateFile.exists()) {
String stateText = FileUtils.readFileToString(stateFile, Charset.defaultCharset()).replace("\n", ""); String stateText = FileUtils.readFileToString(stateFile, Charset.defaultCharset()).replace("\n", "");
testTask.state = TaskState.valueOf(stateText); testTask.state = TaskState.valueOf(stateText);
@@ -319,3 +254,4 @@ public class TestsSupervisor_2022 {
return false; return false;
} }
} }

2
src/Visual_DVM_2021/Passes/All/RemoteInitialiseUser.java
View File

@@ -1,6 +1,5 @@
package Visual_DVM_2021.Passes.All; package Visual_DVM_2021.Passes.All;
import Common.Current; import Common.Current;
import Common.UI.UI;
import Common.Utils.Utils; import Common.Utils.Utils;
import GlobalData.RemoteFile.RemoteFile; import GlobalData.RemoteFile.RemoteFile;
import Visual_DVM_2021.Passes.SSH.CurrentConnectionPass; import Visual_DVM_2021.Passes.SSH.CurrentConnectionPass;
@@ -63,7 +62,6 @@ public class RemoteInitialiseUser extends CurrentConnectionPass<String> {
Files.copy(i, local_p.toPath(), StandardCopyOption.REPLACE_EXISTING); Files.copy(i, local_p.toPath(), StandardCopyOption.REPLACE_EXISTING);
putSingleFile(local_p.getAbsolutePath(), p); putSingleFile(local_p.getAbsolutePath(), p);
} }
UI.Info("+");
//------------------------------------- //-------------------------------------
ShowMessage1("Сборка модулей..."); ShowMessage1("Сборка модулей...");
//канал на исполнение независим, поэтому переход в папку отдельный //канал на исполнение независим, поэтому переход в папку отдельный

2
src/files/Planner/Planner.cpp
View File

@@ -124,7 +124,7 @@ int main(int argc, char ** argv)
printf(">>>>\n"); printf(">>>>\n");
CompilationSupervisor * compilationSupervisor = new CompilationSupervisor(); CompilationSupervisor * compilationSupervisor = new CompilationSupervisor();
printf("%ld\n", compilationSupervisor->getLength()); printf("%ld\n", compilationSupervisor->getLength());
compilationSupervisor->Do(); compilationSupervisor->DoWithSchedule(maxKernels);
RunSupervisor * runSupervisor = new RunSupervisor(compilationSupervisor); RunSupervisor * runSupervisor = new RunSupervisor(compilationSupervisor);
printf("%ld\n", runSupervisor->getLength()); printf("%ld\n", runSupervisor->getLength());

12
src/files/Planner/RunTask.h
View File

@@ -93,18 +93,12 @@ public:
printf("START %ld: %s\n", id, res.getCharArray()); printf("START %ld: %s\n", id, res.getCharArray());
return res; return res;
} }
virtual String copyResults(const String& pathRes) {
virtual void copyResults(const String& pathRes) { String resultPath = Task::copyResults(pathRes);
String resultPath(packageWorkspace + "/" + pathRes + "/" + getId());
Utils::Mkdir(resultPath);
Utils::Copy(workspace + "/out.txt", resultPath + "/out.txt");
Utils::Copy(workspace + "/err.txt", resultPath + "/err.txt");
if (Utils::Exists(workspace + "/sts.gz+")) { if (Utils::Exists(workspace + "/sts.gz+")) {
String dvm_start = String::DQuotes(dvm_drv) + " pa " + String::DQuotes(String(getId()) + "/sts.gz+") + " " + String::DQuotes(resultPath + "/statistic.txt"); String dvm_start = String::DQuotes(dvm_drv) + " pa " + String::DQuotes(String(getId()) + "/sts.gz+") + " " + String::DQuotes(resultPath + "/statistic.txt");
system(dvm_start.getCharArray()); system(dvm_start.getCharArray());
} }
return resultPath;
} }
}; };

61
src/files/Planner/Supervisor.h
View File

@@ -60,7 +60,6 @@ public:
delete packedTasks; delete packedTasks;
delete lines; delete lines;
} }
void changeState() { void changeState() {
switch (this->state) { switch (this->state) {
case WorkspacesCreation: case WorkspacesCreation:
@@ -81,60 +80,8 @@ public:
break; break;
} }
} }
void Do() {
saveState();
long activeCount = 0;
//todo обязательно убрать отладочную печать.
printf("tasks count = %ld\n", this->getLength());
while (this->state != End) {
// printf("state=%d\n", this->state);
// printf("max=%d; busy=%d; free=%d\n", maxKernels, busyKernels, freeKernels);
activeCount = 0;
for (long i = 0; i < this->getLength(); ++i) {
T* task = this->get(i);
switch (this->state) {
case WorkspacesCreation:
if (task->getState() == Waiting) {
activeCount++;
task->createWorkspace();
task->setState(WorkspaceCreated);
}
break;
case Preparation:
if (task->getState() == WorkspaceCreated) {
activeCount++;
task->prepareWorkspace();
task->createLaunchScript();
task->setState(WorkspaceReady);
}
break;
case Execution:
if (task->getState() == WorkspaceReady) {
activeCount++;
task->Start();
}
else if (task->getState() == Running) {
activeCount++;
task->Check();
}
break;
default:
// printf("id = %ld; state = %d\n", task->getId(), task->getState());
break;
}
}
// printf("active count = %d\n", activeCount);
if (activeCount == 0)
changeState();
Utils::Sleep(2);
}
}
void DoWithSchedule(int maxKernels) { void DoWithSchedule(int maxKernels) {
saveState(); saveState();
// подготовка тестов // подготовка тестов
while (this->state != Execution) { while (this->state != Execution) {
for (auto& task : this->getElements()) { for (auto& task : this->getElements()) {
@@ -179,7 +126,7 @@ public:
String pathRes("results"); String pathRes("results");
Utils::Mkdir(pathRes); Utils::Mkdir(pathRes);
string buf; //string buf;
while (activeTasks) { while (activeTasks) {
long oldActiveTasks = activeTasks; long oldActiveTasks = activeTasks;
@@ -227,7 +174,7 @@ public:
busyKernels -= task->getKernels(); busyKernels -= task->getKernels();
printf(" done task with %d kernels and id %ld\n", task->getKernels(), task->getId()); printf(" done task with %d kernels and id %ld\n", task->getKernels(), task->getId());
buf += to_string(task->getId()) + " " + string(task->printState().getCharArray()) + " " + to_string(task->getTotalTime()) + "\n"; //buf += to_string(task->getId()) + " " + string(task->printState().getCharArray()) + " " + to_string(task->getTotalTime()) + "\n";
task->copyResults(pathRes); task->copyResults(pathRes);
continue; continue;
} }
@@ -240,8 +187,8 @@ public:
changeState(); changeState();
String outFile(pathRes + "/info.txt"); //String outFile(pathRes + "/info.txt");
File tmp(outFile, String(buf.c_str())); //File tmp(outFile, String(buf.c_str()));
Utils::ZipFolder(pathRes, pathRes + ".zip"); Utils::ZipFolder(pathRes, pathRes + ".zip");
} }

10
src/files/Planner/Task.h
View File

@@ -198,5 +198,13 @@ public:
File tmp(stateFile, printState()); File tmp(stateFile, printState());
} }
virtual void copyResults(const String& path) { } virtual String copyResults(const String& pathRes) {
String resultPath(packageWorkspace + "/" + pathRes + "/" + getId());
Utils::Mkdir(resultPath);
Utils::Copy(workspace + "/TaskState", resultPath + "/TaskState");
Utils::Copy(workspace + "/out.txt", resultPath + "/out.txt");
Utils::Copy(workspace + "/err.txt", resultPath + "/err.txt");
Utils::Copy(workspace + "/total_time", resultPath + "/total_time");
return resultPath;
}
}; };