Update the AskSage method.

The code that is submitted via the AskSage method no longer works with
the newer versions of Sage and needs to be updated.

Also the public Sage cell server at https://sagecell.sagemath.org/service
no longer accepts public code via its `service` endpoint.  So make the
URL for a sagecell server configurable.

Files for building a sagecell server and deploying it via docker have
been added in the `fsagecell-docker` directory. Instructions for doing
so are in the `sagecell-docker/README.md` file.

Remove the `accepted_tos` parameter from the `AskSage` call.  Just add
the parameter internally, and stop requiring that the author do so.  It
is a meaningless gesture to force the author to do so.  Furthermore, the
docker build is set to not require that parameter.

Also make it so that the `AskSage` method can be called without passing
any arguments, since there are now none that are required.

Author: drgrice1

.gitignore

@@ -10,3 +10,5 @@ htdocs/static-assets.json
 htdocs/**/*.min.js
 htdocs/**/*.min.css
 htdocs/tmp
+
+sagecell-docker/sagecell-docker.service

conf/pg_config.dist.yml

@@ -74,6 +74,17 @@ URLs:
     - .
     - $pg_root_url/images
 
+  # By default the AskSage method uses the public sagecell service located at
+  # https://sagecell.sagemath.org/service. However, the server at that address
+  # is no longer accepting public requests. So in order for problems that use
+  # the AskSage method to work a different sagecell server must be used.
+  # Uncomment the line below and set the value to a custom sagecell server
+  # address to use with the AskSage method.  If using the docker container built
+  # with the instructions and files in the sagecell-docker directory in this
+  # repository, the use the line below as is.
+
+  #sagecellService: http://localhost:3500/service
+
 # Flat-file database used to protect against MD5 hash collisions. TeX equations
 # are hashed to determine the name of the image file. There is a tiny chance of
 # a collision between two TeX strings. This file allows for that. However, this

lib/PGcore.pm

@@ -703,9 +703,8 @@ sub directoryFromPath {
 }
 
 sub AskSage {
-	my $self    = shift;
-	my $python  = shift;
-	my $options = shift;
+	my ($self, $python, $options) = @_;
+	$options = {} unless ref $options eq 'HASH';
 	$options->{curlCommand} = WeBWorK::PG::IO::externalCommand('curl');
 	WeBWorK::PG::IO::AskSage($python, $options);
 }

lib/WeBWorK/PG/IO.pm

@@ -315,7 +315,7 @@ sub externalCommand {
 
 # Isolate the call to the sage server in case we have to jazz it up.
 sub query_sage_server {
-	my ($python, $url, $accepted_tos, $setSeed, $webworkfunc, $debug, $curlCommand) = @_;
+	my ($python, $url, $setSeed, $webworkfunc, $debug, $curlCommand) = @_;
 
 	# Validate url to prevent shell injection — must look like a URL.
 	if ($url && $url !~ m{^https?://[^\s;|&`\$]+$}) {
@@ -328,7 +328,7 @@ sub query_sage_server {
 		$curlCommand,                                                 '-i',
 		'-k',                                                         '-sS',
 		'-L',                                                         '--data-urlencode',
-		"accepted_tos=$accepted_tos",                                 '--data-urlencode',
+		"accepted_tos=true",                                          '--data-urlencode',
 		'user_expressions={"WEBWORK":"_webwork_safe_json(WEBWORK)"}', '--data-urlencode',
 		"code=${setSeed}${webworkfunc}$python",                       $url // (),
 	);
@@ -400,8 +400,7 @@ sub query_sage_server {
 		# Success! Put any extraneous splits back together.
 		$result = join("\r\n\r\n", @lines);
 	} else {
-		warn "ERROR in contacting sage server. Did you accept the terms of service by "
-			. "setting { accepted_tos => 'true' } in the askSage options?\n$content\n";
+		warn "ERROR in contacting sage server.\n$content\n";
 		$result = undef;
 	}
 
@@ -421,44 +420,44 @@ sub AskSage {
 	chomp($python);
 
 	# To send values back in a hash, add them to the python WEBWORK dictionary.
-	my $url          = $args->{url} || 'https://sagecell.sagemath.org/service';
-	my $seed         = $args->{seed};
-	my $accepted_tos = $args->{accepted_tos} || 'false';    # Force author to accept terms of service explicitly.
-	my $debug        = $args->{debug}        || 0;
-	my $setSeed      = $seed ? "set_random_seed($seed)\n" : '';
-	my $curlCommand  = $args->{curlCommand};
+	my $url         = $args->{url} || $pg_envir->{URLs}{sagecellService} || 'https://sagecell.sagemath.org/service';
+	my $seed        = $args->{seed};
+	my $debug       = $args->{debug} || 0;
+	my $setSeed     = $seed ? "set_random_seed($seed)\n" : '';
+	my $curlCommand = $args->{curlCommand};
 
 	my $webworkfunc = <<END;
 WEBWORK={}
 
 def _webwork_safe_json(o):
     import json
-    def default(o):
-        try:
-            if isinstance(o,sage.rings.integer.Integer):
-                json_obj = int(o)
-            elif isinstance(o,(sage.rings.real_mpfr.RealLiteral, sage.rings.real_mpfr.RealNumber)):
-                json_obj = float(o)
-            elif sage.modules.free_module_element.is_FreeModuleElement(o):
-                json_obj = list(o)
-            elif sage.matrix.matrix.is_Matrix(o):
-                json_obj = [list(i) for i in o.rows()]
-            elif isinstance(o, SageObject):
-                json_obj = repr(o)
+    class WebworkEncoder(json.JSONEncoder):
+        def default(self, o):
+            try:
+                if isinstance(o,sage.rings.integer.Integer):
+                    json_obj = int(o)
+                elif isinstance(o,(sage.rings.real_mpfr.RealLiteral, sage.rings.real_mpfr.RealNumber)):
+                    json_obj = float(o)
+                elif isinstance(o, sage.modules.free_module_element.FreeModuleElement):
+                    json_obj = list(o)
+                elif isinstance(o, sage.matrix.matrix0.Matrix):
+                    json_obj = [list(i) for i in o.rows()]
+                elif isinstance(o, SageObject):
+                    json_obj = repr(o)
+                else:
+                    raise TypeError
+            except TypeError:
+                pass
             else:
-                raise TypeError
-        except TypeError:
-            pass
-        else:
-            return json_obj
-        # Let the base class default method raise the TypeError
-        return json.JSONEncoder.default(self, o)
-    return json.dumps(o, default=default)
+                return json_obj
+            # Let the base class default method raise the TypeError
+            return super().default(o)
+    return json.dumps(o, cls=WebworkEncoder)
 END
 
 	my $ret = { success => 0 };    # We want to export more than one piece of information.
 	eval {
-		my $output = query_sage_server($python, $url, $accepted_tos, $setSeed, $webworkfunc, $debug, $curlCommand);
+		my $output = query_sage_server($python, $url, $setSeed, $webworkfunc, $debug, $curlCommand);
 
 		# has something been returned?
 		not_null($output) or die "Unable to make a sage call to $url.";
@@ -486,7 +485,7 @@ END
 		warn "now success  is $success because status was ok" if $debug;
 		if ($success) {
 			my $WEBWORK_variable_non_empty = 0;
-			my $sage_WEBWORK_data          = $decoded->{execute_reply}{user_expressions}{WEBWORK}{data}{'text/plain'};
+			my $sage_WEBWORK_data = $decoded->{execute_reply}{user_expressions}{WEBWORK}{data}{'text/plain'} // '';
 			warn "sage_WEBWORK_data $sage_WEBWORK_data" if $debug;
 			if (not_null($sage_WEBWORK_data)) {
 				$WEBWORK_variable_non_empty =    # another hack because '{}' is sometimes returned

macros/PG.pl

@@ -397,9 +397,7 @@ ()
 }
 
 sub AskSage {
-	my $python  = shift;
-	my $options = shift;
-	WARN_MESSAGE("the second argument to AskSage should be a hash of options") unless $options =~ /HASH/;
+	my ($python, $options) = @_;
 	$PG->AskSage($python, $options);
 }
 

sagecell-docker/Dockerfile

@@ -0,0 +1,63 @@
+FROM sagemath/sagemath:10.8
+
+USER root
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN=true
+ENV DEBCONF_NOWARNINGS=yes
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    automake \
+    build-essential \
+    curl \
+    git \
+    locales \
+    openssh-server \
+    python3 \
+    python-is-python3 \
+    python3-jupyter-client \
+    python3-pip \
+    python3-psutil \
+    python3-zmq \
+    rsyslog \
+    wget \
+    && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
+    && locale-gen \
+    && curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \
+    && apt-get install -y --no-install-recommends --no-install-suggests nodejs
+
+ENV SHELL=/bin/bash
+
+USER sage
+
+RUN echo "Installing SageCell server for SageMath" \
+    && sage -pip install lockfile paramiko sockjs-tornado sqlalchemy \
+    && git clone --single-branch --branch master --depth 1 https://github.com/sagemath/sagecell.git \
+    && cd sagecell \
+    && { sage -sh -c make || true; } \
+    && sage -sh -c make \
+    && rm -rf contrib doc tests .git \
+    && cp config_default.py config.py \
+    && sed -i 's/"username": None/"username": "sage"/' config.py \
+	&& sed -i 's/requires_tos = True/requires_tos = False/' config.py
+
+USER root
+
+COPY sagecell-entrypoint.sh /usr/local/bin/sagecell-entrypoint
+
+RUN echo "Configuring internal ssh server" \
+    && chmod +x /usr/local/bin/sagecell-entrypoint \
+    && echo "ListenAddress 127.0.0.1\nPasswordAuthentication no" \
+        > /etc/ssh/sshd_config.d/sshd_config.conf
+
+RUN echo "Cleaning the Container" \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/*
+
+ENV PORT=3500
+
+WORKDIR /home/sage/sagecell
+
+ENTRYPOINT ["sagecell-entrypoint"]
+CMD ["sudo", "-H", "-E", "-u", "sage", "sage", "web_server.py", "-p $PORT"]

sagecell-docker/README.md

@@ -0,0 +1,47 @@
+# SageCell Docker Instructions
+
+These are instructions to build and use a docker image for running a SageCell service in production.
+
+## Build and Run the Docker Image
+
+Execute the following command from the `sagecell-docker` directory.
+
+```bash
+docker build -t sagecell .
+```
+
+Then run the container by executing the following command in the `sagecell-docker` directory.
+
+```bash
+docker run --rm -p 127.0.0.1:3500:3500 sagecell:latest
+```
+
+Note that the `-p` argument exposes port `3500` inside the container to port `3500` outside the container, but only
+allows connections from localhost. This is important as it does not allow anyone else to use the SageCell service other
+than the local webwork server. If a different port outside the container is needed, then change the first `3500` to the
+desired port. Adding `--rm` removes the created volume for the container when it exits. Exit the container by typing
+`Ctrl-C`.
+
+Access by specific external IP addresses is also possible if you want to have the SageCell service on another server.
+The easiest way to accomplish this is by using an SSH tunnel. For example, execute
+`ssh -L 3500:127.0.0.1:3550 userId@sagecell.server` where `userId` is your username on the SageCell server and
+`sagecell.server` is the domain name or IP address of the SageCell server. There are other ways to accomplish this as
+well, but the important thing is that access to the SageCell service be highly restricted as it allows untrusted code to
+be executed.
+
+## Deploy the Image for Production Usage
+
+Copy the file `sagecell-docker.dist.service` to `sagecell-docker.service` and execute the following command from the
+`sagecell-docker` directory to enable a `systemd` service for running the container.
+
+```bash
+sudo systemctl enable $(pwd)/sagecell-docker.service
+```
+
+Then you can start the container by executing `sudo systemctl start sagecell-docker`, and stop the container by
+executing `sudo systemctl stop sagecell-docker`. Note that any time the server is rebooted the service will start
+automatically, so you usually do not need to execute those commands other than to start the service the first time after
+enabling it.
+
+Note that the service simply executes the command give above for running the container directly. So modify the command
+in the `sagecell-docker.service` file as described above if a different port is needed.

sagecell-docker/sagecell-docker.dist.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=sagecell server
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=docker run --rm -p 127.0.0.1:3500:3500 sagecell:latest
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target

sagecell-docker/sagecell-entrypoint.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+function configure_ssh() {
+	local key_file=/home/sage/.ssh/id_rsa
+	if [ $(ssh-add -l &> /dev/null; echo $?) -gt 0 ]; then
+		# Generate ssh keys for the user.
+		if [ ! -f $key_file ]; then
+			[ ! -d /home/sage/.ssh ] && mkdir /home/sage/.ssh
+			ssh-keygen -t rsa -f $key_file -P "" -q
+		fi
+
+		# Start and configure the ssh-agent.
+		eval `ssh-agent` > /dev/null
+		local ssh_fingerprint=$(ssh-keygen -l -f $key_file | awk '{print $2}')
+		if [ ! -z  "$ssh_fingerprint" ] && [ -z "$(ssh-add -l | grep "$ssh_fingerprint")" ]; then
+			ssh-add $key_file 2> /dev/null
+		fi
+
+		# Add localhost to known_hosts.
+		[ ! -f /home/sage/.ssh/known_hosts ] && touch /home/sage/.ssh/known_hosts
+		if [ -z "$(ssh-keygen -F localhost)" ]; then
+			ssh-keyscan -H localhost > /home/sage/.ssh/known_hosts 2> /dev/null
+		fi
+
+		cat ${key_file}.pub > /home/sage/.ssh/authorized_keys
+	fi
+}
+
+function check_ssh() {
+	ssh -v -o PreferredAuthentications=publickey -o BatchMode=yes -o ConnectTimeout=10 \
+		-o StrictHostKeyChecking=no localhost /bin/true &> /dev/null
+	return $?
+}
+
+# Configure and start rsyslog.
+sed -i '/imklog/s/^/#/' /etc/rsyslog.conf
+rsyslogd
+
+# Start and configure the ssh server for internal use.
+service ssh start > /dev/null
+
+export -f configure_ssh
+export -f check_ssh
+
+su sage bash -c "configure_ssh"
+su sage bash -c "check_ssh"
+
+if [ $? -eq 0 ]; then
+    echo "Executing: $(eval "echo $@")"
+    exec $(eval "echo $@")
+else
+    echo "ERROR: SageCell server not started. Failed to configure internal ssh server."
+fi